functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import re
  15 import subprocess
  16 import sys
  17 import fileinput
  18
  19 import yaml
  20
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23 from functest.utils.constants import CONST
  24
  25 from opnfv.utils import constants as opnfv_constants
  26 from opnfv.deployment import factory
  27
  28 actions = ['start', 'check']
  29
  30 """ logging configuration """
  31 logger = logging.getLogger('functest.ci.prepare_env')
  32 handler = None
  33 # set the architecture to default
  34 pod_arch = None
  35 arch_filter = ['aarch64']
  36
  37 CONFIG_FUNCTEST_PATH = CONST.__getattribute__('CONFIG_FUNCTEST_YAML')
  38 CONFIG_PATCH_PATH = os.path.join(os.path.dirname(
  39     CONFIG_FUNCTEST_PATH), "config_patch.yaml")
  40 CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  41     CONFIG_FUNCTEST_PATH), "config_aarch64_patch.yaml")
  42 RALLY_CONF_PATH = os.path.join("/etc/rally/rally.conf")
  43 RALLY_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  44     CONFIG_FUNCTEST_PATH), "rally_aarch64_patch.conf")
  45
  46
  47 class PrepareEnvParser(object):
  48
  49     def __init__(self):
  50         self.parser = argparse.ArgumentParser()
  51         self.parser.add_argument("action", help="Possible actions are: "
  52                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  53                                  choices=actions)
  54         self.parser.add_argument("-d", "--debug", help="Debug mode",
  55                                  action="store_true")
  56
  57     def parse_args(self, argv=[]):
  58         return vars(self.parser.parse_args(argv))
  59
  60
  61 def print_separator():
  62     logger.info("==============================================")
  63
  64
  65 def check_env_variables():
  66     print_separator()
  67     logger.info("Checking environment variables...")
  68
  69     if CONST.__getattribute__('INSTALLER_TYPE') is None:
  70         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  71         CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  72     else:
  73         if (CONST.__getattribute__('INSTALLER_TYPE') not in
  74                 opnfv_constants.INSTALLERS):
  75             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  76                            "Available OPNFV Installers are : %s. "
  77                            "Setting INSTALLER_TYPE=undefined."
  78                            % (CONST.__getattribute__('INSTALLER_TYPE'),
  79                               opnfv_constants.INSTALLERS))
  80             CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  81         else:
  82             logger.info("    INSTALLER_TYPE=%s"
  83                         % CONST.__getattribute__('INSTALLER_TYPE'))
  84
  85     if CONST.__getattribute__('INSTALLER_IP') is None:
  86         logger.warning("The env variable 'INSTALLER_IP' is not defined. "
  87                        "It is needed to fetch the OpenStack credentials. "
  88                        "If the credentials are not provided to the "
  89                        "container as a volume, please add this env variable "
  90                        "to the 'docker run' command.")
  91     else:
  92         logger.info("    INSTALLER_IP=%s" %
  93                     CONST.__getattribute__('INSTALLER_IP'))
  94
  95     if CONST.__getattribute__('DEPLOY_SCENARIO') is None:
  96         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  97                        "Setting CI_SCENARIO=undefined.")
  98         CONST.__setattr__('DEPLOY_SCENARIO', 'undefined')
  99     else:
 100         logger.info("    DEPLOY_SCENARIO=%s"
 101                     % CONST.__getattribute__('DEPLOY_SCENARIO'))
 102     if CONST.__getattribute__('CI_DEBUG'):
 103         logger.info("    CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG'))
 104
 105     if CONST.__getattribute__('NODE_NAME'):
 106         logger.info("    NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME'))
 107
 108     if CONST.__getattribute__('BUILD_TAG'):
 109         logger.info("    BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG'))
 110
 111     if CONST.__getattribute__('IS_CI_RUN'):
 112         logger.info("    IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN'))
 113
 114
 115 def get_deployment_handler():
 116     global handler
 117     global pod_arch
 118
 119     installer_params_yaml = os.path.join(
 120         CONST.__getattribute__('dir_repo_functest'),
 121         'functest/ci/installer_params.yaml')
 122     if (CONST.__getattribute__('INSTALLER_IP') and
 123         CONST.__getattribute__('INSTALLER_TYPE') and
 124             CONST.__getattribute__('INSTALLER_TYPE') in
 125             opnfv_constants.INSTALLERS):
 126         try:
 127             installer_params = ft_utils.get_parameter_from_yaml(
 128                 CONST.__getattribute__('INSTALLER_TYPE'),
 129                 installer_params_yaml)
 130         except ValueError as e:
 131             logger.debug('Printing deployment info is not supported for %s' %
 132                          CONST.__getattribute__('INSTALLER_TYPE'))
 133             logger.debug(e)
 134         else:
 135             user = installer_params.get('user', None)
 136             password = installer_params.get('password', None)
 137             pkey = installer_params.get('pkey', None)
 138             try:
 139                 handler = factory.Factory.get_handler(
 140                     installer=CONST.__getattribute__('INSTALLER_TYPE'),
 141                     installer_ip=CONST.__getattribute__('INSTALLER_IP'),
 142                     installer_user=user,
 143                     installer_pwd=password,
 144                     pkey_file=pkey)
 145                 if handler:
 146                     pod_arch = handler.get_arch()
 147             except Exception as e:
 148                 logger.debug("Cannot get deployment information. %s" % e)
 149
 150
 151 def create_directories():
 152     print_separator()
 153     logger.info("Creating needed directories...")
 154     if not os.path.exists(CONST.__getattribute__('dir_functest_conf')):
 155         os.makedirs(CONST.__getattribute__('dir_functest_conf'))
 156         logger.info("    %s created." %
 157                     CONST.__getattribute__('dir_functest_conf'))
 158     else:
 159         logger.debug("   %s already exists." %
 160                      CONST.__getattribute__('dir_functest_conf'))
 161
 162     if not os.path.exists(CONST.__getattribute__('dir_functest_data')):
 163         os.makedirs(CONST.__getattribute__('dir_functest_data'))
 164         logger.info("    %s created." %
 165                     CONST.__getattribute__('dir_functest_data'))
 166     else:
 167         logger.debug("   %s already exists." %
 168                      CONST.__getattribute__('dir_functest_data'))
 169     if not os.path.exists(CONST.__getattribute__('dir_functest_images')):
 170         os.makedirs(CONST.__getattribute__('dir_functest_images'))
 171         logger.info("    %s created." %
 172                     CONST.__getattribute__('dir_functest_images'))
 173     else:
 174         logger.debug("   %s already exists." %
 175                      CONST.__getattribute__('dir_functest_images'))
 176
 177
 178 def source_rc_file():
 179     print_separator()
 180     logger.info("Fetching RC file...")
 181
 182     if CONST.__getattribute__('openstack_creds') is None:
 183         logger.warning("The environment variable 'creds' must be set and"
 184                        "pointing to the local RC file. Using default: "
 185                        "/home/opnfv/functest/conf/openstack.creds ...")
 186         os.path.join(
 187             CONST.__getattribute__('dir_functest_conf'), 'openstack.creds')
 188
 189     if not os.path.isfile(CONST.__getattribute__('openstack_creds')):
 190         logger.info("RC file not provided. "
 191                     "Fetching it from the installer...")
 192         if CONST.__getattribute__('INSTALLER_IP')is None:
 193             logger.error("The env variable 'INSTALLER_IP' must be provided in"
 194                          " order to fetch the credentials from the installer.")
 195             raise Exception("Missing CI_INSTALLER_IP.")
 196         if (CONST.__getattribute__('INSTALLER_TYPE') not in
 197                 opnfv_constants.INSTALLERS):
 198             logger.error("Cannot fetch credentials. INSTALLER_TYPE=%s is "
 199                          "not a valid OPNFV installer. Available "
 200                          "installers are : %s." %
 201                          (CONST.__getattribute__('INSTALLER_TYPE'),
 202                           opnfv_constants.INSTALLERS))
 203             raise Exception("Wrong INSTALLER_TYPE.")
 204
 205         cmd = ("/home/opnfv/repos/releng/utils/fetch_os_creds.sh "
 206                "-d %s -i %s -a %s"
 207                % (CONST.__getattribute__('openstack_creds'),
 208                   CONST.__getattribute__('INSTALLER_TYPE'),
 209                   CONST.__getattribute__('INSTALLER_IP')))
 210         logger.debug("Executing command: %s" % cmd)
 211         p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
 212         output = p.communicate()[0]
 213         logger.debug("\n%s" % output)
 214         if p.returncode != 0:
 215             raise Exception("Failed to fetch credentials from installer.")
 216     else:
 217         logger.info("RC file provided in %s."
 218                     % CONST.__getattribute__('openstack_creds'))
 219         if os.path.getsize(CONST.__getattribute__('openstack_creds')) == 0:
 220             raise Exception("The file %s is empty." %
 221                             CONST.__getattribute__('openstack_creds'))
 222
 223     logger.info("Sourcing the OpenStack RC file...")
 224     os_utils.source_credentials(CONST.__getattribute__('openstack_creds'))
 225     for key, value in os.environ.iteritems():
 226         if re.search("OS_", key):
 227             if key == 'OS_AUTH_URL':
 228                 CONST.__setattr__('OS_AUTH_URL', value)
 229             elif key == 'OS_USERNAME':
 230                 CONST.__setattr__('OS_USERNAME', value)
 231             elif key == 'OS_TENANT_NAME':
 232                 CONST.__setattr__('OS_TENANT_NAME', value)
 233             elif key == 'OS_PASSWORD':
 234                 CONST.__setattr__('OS_PASSWORD', value)
 235
 236
 237 def update_config_file():
 238     patch_file(CONFIG_PATCH_PATH)
 239
 240     if pod_arch and pod_arch in arch_filter:
 241         patch_file(CONFIG_AARCH64_PATCH_PATH)
 242
 243     if "TEST_DB_URL" in os.environ:
 244         update_db_url()
 245
 246
 247 def patch_file(patch_file_path):
 248     logger.debug('Updating file: %s', patch_file_path)
 249     with open(patch_file_path) as f:
 250         patch_file = yaml.safe_load(f)
 251
 252     updated = False
 253     for key in patch_file:
 254         if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
 255             new_functest_yaml = dict(ft_utils.merge_dicts(
 256                 ft_utils.get_functest_yaml(), patch_file[key]))
 257             updated = True
 258
 259     if updated:
 260         os.remove(CONFIG_FUNCTEST_PATH)
 261         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 262             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 263
 264
 265 def update_db_url():
 266     with open(CONFIG_FUNCTEST_PATH) as f:
 267         functest_yaml = yaml.safe_load(f)
 268
 269     with open(CONFIG_FUNCTEST_PATH, "w") as f:
 270         functest_yaml["results"]["test_db_url"] = os.environ.get('TEST_DB_URL')
 271         f.write(yaml.dump(functest_yaml, default_style='"'))
 272
 273
 274 def verify_deployment():
 275     print_separator()
 276     logger.info("Verifying OpenStack services...")
 277     cmd = ("%s/functest/ci/check_os.sh" %
 278            CONST.__getattribute__('dir_repo_functest'))
 279
 280     logger.debug("Executing command: %s" % cmd)
 281     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 282
 283     while p.poll() is None:
 284         line = p.stdout.readline().rstrip()
 285         if "ERROR" in line:
 286             logger.error(line)
 287             raise Exception("Problem while running 'check_os.sh'.")
 288         logger.info(line)
 289
 290
 291 def install_rally():
 292     print_separator()
 293
 294     if pod_arch and pod_arch in arch_filter:
 295         logger.info("Apply aarch64 specific to rally config...")
 296         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 297             rally_patch_conf = f.read()
 298
 299         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 300             print line,
 301             if "cirros|testvm" in line:
 302                 print rally_patch_conf
 303
 304     logger.info("Creating Rally environment...")
 305
 306     cmd = "rally deployment destroy opnfv-rally"
 307     ft_utils.execute_command(cmd, error_msg=(
 308         "Deployment %s does not exist."
 309         % CONST.__getattribute__('rally_deployment_name')),
 310         verbose=False)
 311
 312     rally_conf = os_utils.get_credentials_for_rally()
 313     with open('rally_conf.json', 'w') as fp:
 314         json.dump(rally_conf, fp)
 315     cmd = ("rally deployment create "
 316            "--file=rally_conf.json --name={0}"
 317            .format(CONST.__getattribute__('rally_deployment_name')))
 318     error_msg = "Problem while creating Rally deployment"
 319     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 320
 321     cmd = "rally deployment check"
 322     error_msg = "OpenStack not responding or faulty Rally deployment."
 323     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 324
 325     cmd = "rally deployment list"
 326     ft_utils.execute_command(cmd,
 327                              error_msg=("Problem while listing "
 328                                         "Rally deployment."))
 329
 330     cmd = "rally plugin list | head -5"
 331     ft_utils.execute_command(cmd,
 332                              error_msg=("Problem while showing "
 333                                         "Rally plugins."))
 334
 335
 336 def install_tempest():
 337     logger.info("Installing tempest from existing repo...")
 338     cmd = ("rally verify list-verifiers | "
 339            "grep '{0}' | wc -l".format(
 340                CONST.__getattribute__('tempest_deployment_name')))
 341     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 342     while p.poll() is None:
 343         line = p.stdout.readline().rstrip()
 344         if str(line) == '0':
 345             logger.debug("Tempest %s does not exist" %
 346                          CONST.__getattribute__('tempest_deployment_name'))
 347             cmd = ("rally verify create-verifier --source {0} "
 348                    "--name {1} --type tempest --system-wide"
 349                    .format(CONST.__getattribute__('dir_repo_tempest'),
 350                            CONST.__getattribute__('tempest_deployment_name')))
 351             error_msg = "Problem while installing Tempest."
 352             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 353
 354
 355 def create_flavor():
 356     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 357                                                  '512',
 358                                                  '1',
 359                                                  '1',
 360                                                  public=True)
 361     if flavor_id is None:
 362         raise Exception('Failed to create flavor')
 363
 364
 365 def check_environment():
 366     msg_not_active = "The Functest environment is not installed."
 367     if not os.path.isfile(CONST.__getattribute__('env_active')):
 368         raise Exception(msg_not_active)
 369
 370     with open(CONST.__getattribute__('env_active'), "r") as env_file:
 371         s = env_file.read()
 372         if not re.search("1", s):
 373             raise Exception(msg_not_active)
 374
 375     logger.info("Functest environment is installed.")
 376
 377
 378 def print_deployment_info():
 379     if handler:
 380         logger.info('\n\nDeployment information:\n%s' %
 381                     handler.get_deployment_info())
 382
 383
 384 def main(**kwargs):
 385     try:
 386         if not (kwargs['action'] in actions):
 387             logger.error('Argument not valid.')
 388             return -1
 389         elif kwargs['action'] == "start":
 390             logger.info("######### Preparing Functest environment #########\n")
 391             check_env_variables()
 392             get_deployment_handler()
 393             create_directories()
 394             source_rc_file()
 395             update_config_file()
 396             verify_deployment()
 397             install_rally()
 398             install_tempest()
 399             create_flavor()
 400             with open(CONST.__getattribute__('env_active'), "w") as env_file:
 401                 env_file.write("1")
 402             check_environment()
 403             print_deployment_info()
 404         elif kwargs['action'] == "check":
 405             check_environment()
 406     except Exception as e:
 407         logger.error(e)
 408         return -1
 409     return 0
 410
 411
 412 if __name__ == '__main__':
 413     logging.config.fileConfig(
 414         CONST.__getattribute__('dir_functest_logging_cfg'))
 415     parser = PrepareEnvParser()
 416     args = parser.parse_args(sys.argv[1:])
 417     sys.exit(main(**args))