functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import re
  15 import subprocess
  16 import sys
  17 import fileinput
  18
  19 import yaml
  20
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23 from functest.utils.constants import CONST
  24
  25 from opnfv.utils import constants as opnfv_constants
  26 from opnfv.deployment import factory
  27
  28 actions = ['start', 'check']
  29
  30 """ logging configuration """
  31 logger = logging.getLogger('functest.ci.prepare_env')
  32 handler = None
  33 # set the architecture to default
  34 pod_arch = None
  35 arch_filter = ['aarch64']
  36
  37 CONFIG_FUNCTEST_PATH = CONST.__getattribute__('CONFIG_FUNCTEST_YAML')
  38 CONFIG_PATCH_PATH = os.path.join(os.path.dirname(
  39     CONFIG_FUNCTEST_PATH), "config_patch.yaml")
  40 CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  41     CONFIG_FUNCTEST_PATH), "config_aarch64_patch.yaml")
  42 RALLY_CONF_PATH = os.path.join("/etc/rally/rally.conf")
  43 RALLY_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  44     CONFIG_FUNCTEST_PATH), "rally_aarch64_patch.conf")
  45
  46
  47 class PrepareEnvParser(object):
  48
  49     def __init__(self):
  50         self.parser = argparse.ArgumentParser()
  51         self.parser.add_argument("action", help="Possible actions are: "
  52                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  53                                  choices=actions)
  54         self.parser.add_argument("-d", "--debug", help="Debug mode",
  55                                  action="store_true")
  56
  57     def parse_args(self, argv=[]):
  58         return vars(self.parser.parse_args(argv))
  59
  60
  61 def print_separator():
  62     logger.info("==============================================")
  63
  64
  65 def check_env_variables():
  66     print_separator()
  67     logger.info("Checking environment variables...")
  68
  69     if CONST.__getattribute__('INSTALLER_TYPE') is None:
  70         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  71         CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  72     else:
  73         if (CONST.__getattribute__('INSTALLER_TYPE') not in
  74                 opnfv_constants.INSTALLERS):
  75             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  76                            "Available OPNFV Installers are : %s. "
  77                            "Setting INSTALLER_TYPE=undefined."
  78                            % (CONST.__getattribute__('INSTALLER_TYPE'),
  79                               opnfv_constants.INSTALLERS))
  80             CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  81         else:
  82             logger.info("    INSTALLER_TYPE=%s"
  83                         % CONST.__getattribute__('INSTALLER_TYPE'))
  84
  85     if CONST.__getattribute__('INSTALLER_IP') is None:
  86         logger.warning("The env variable 'INSTALLER_IP' is not defined. "
  87                        "It is needed to fetch the OpenStack credentials. "
  88                        "If the credentials are not provided to the "
  89                        "container as a volume, please add this env variable "
  90                        "to the 'docker run' command.")
  91     else:
  92         logger.info("    INSTALLER_IP=%s" %
  93                     CONST.__getattribute__('INSTALLER_IP'))
  94
  95     if CONST.__getattribute__('DEPLOY_SCENARIO') is None:
  96         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  97                        "Setting CI_SCENARIO=undefined.")
  98         CONST.__setattr__('DEPLOY_SCENARIO', 'undefined')
  99     else:
 100         logger.info("    DEPLOY_SCENARIO=%s"
 101                     % CONST.__getattribute__('DEPLOY_SCENARIO'))
 102     if CONST.__getattribute__('CI_DEBUG'):
 103         logger.info("    CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG'))
 104
 105     if CONST.__getattribute__('NODE_NAME'):
 106         logger.info("    NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME'))
 107
 108     if CONST.__getattribute__('BUILD_TAG'):
 109         logger.info("    BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG'))
 110
 111     if CONST.__getattribute__('IS_CI_RUN'):
 112         logger.info("    IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN'))
 113
 114
 115 def get_deployment_handler():
 116     global handler
 117     global pod_arch
 118
 119     installer_params_yaml = os.path.join(
 120         CONST.__getattribute__('dir_repo_functest'),
 121         'functest/ci/installer_params.yaml')
 122     if (CONST.__getattribute__('INSTALLER_IP') and
 123         CONST.__getattribute__('INSTALLER_TYPE') and
 124             CONST.__getattribute__('INSTALLER_TYPE') in
 125             opnfv_constants.INSTALLERS):
 126         try:
 127             installer_params = ft_utils.get_parameter_from_yaml(
 128                 CONST.__getattribute__('INSTALLER_TYPE'),
 129                 installer_params_yaml)
 130         except ValueError as e:
 131             logger.debug('Printing deployment info is not supported for %s' %
 132                          CONST.__getattribute__('INSTALLER_TYPE'))
 133             logger.debug(e)
 134         else:
 135             user = installer_params.get('user', None)
 136             password = installer_params.get('password', None)
 137             pkey = installer_params.get('pkey', None)
 138             try:
 139                 handler = factory.Factory.get_handler(
 140                     installer=CONST.__getattribute__('INSTALLER_TYPE'),
 141                     installer_ip=CONST.__getattribute__('INSTALLER_IP'),
 142                     installer_user=user,
 143                     installer_pwd=password,
 144                     pkey_file=pkey)
 145                 if handler:
 146                     pod_arch = handler.get_arch()
 147             except Exception as e:
 148                 logger.debug("Cannot get deployment information. %s" % e)
 149
 150
 151 def create_directories():
 152     print_separator()
 153     logger.info("Creating needed directories...")
 154     if not os.path.exists(CONST.__getattribute__('dir_functest_conf')):
 155         os.makedirs(CONST.__getattribute__('dir_functest_conf'))
 156         logger.info("    %s created." %
 157                     CONST.__getattribute__('dir_functest_conf'))
 158     else:
 159         logger.debug("   %s already exists."
 160                      % CONST.__getattribute__('dir_functest_conf'))
 161
 162     if not os.path.exists(CONST.__getattribute__('dir_functest_data')):
 163         os.makedirs(CONST.__getattribute__('dir_functest_data'))
 164         logger.info("    %s created." %
 165                     CONST.__getattribute__('dir_functest_data'))
 166     else:
 167         logger.debug("   %s already exists." %
 168                      CONST.__getattribute__('dir_functest_data'))
 169
 170
 171 def source_rc_file():
 172     print_separator()
 173     logger.info("Fetching RC file...")
 174
 175     if CONST.__getattribute__('openstack_creds') is None:
 176         logger.warning("The environment variable 'creds' must be set and"
 177                        "pointing to the local RC file. Using default: "
 178                        "/home/opnfv/functest/conf/openstack.creds ...")
 179         os.path.join(
 180             CONST.__getattribute__('dir_functest_conf'), 'openstack.creds')
 181
 182     if not os.path.isfile(CONST.__getattribute__('openstack_creds')):
 183         logger.info("RC file not provided. "
 184                     "Fetching it from the installer...")
 185         if CONST.__getattribute__('INSTALLER_IP')is None:
 186             logger.error("The env variable 'INSTALLER_IP' must be provided in"
 187                          " order to fetch the credentials from the installer.")
 188             raise Exception("Missing CI_INSTALLER_IP.")
 189         if (CONST.__getattribute__('INSTALLER_TYPE') not in
 190                 opnfv_constants.INSTALLERS):
 191             logger.error("Cannot fetch credentials. INSTALLER_TYPE=%s is "
 192                          "not a valid OPNFV installer. Available "
 193                          "installers are : %s." %
 194                          (CONST.__getattribute__('INSTALLER_TYPE'),
 195                           opnfv_constants.INSTALLERS))
 196             raise Exception("Wrong INSTALLER_TYPE.")
 197
 198         cmd = ("/home/opnfv/repos/releng/utils/fetch_os_creds.sh "
 199                "-d %s -i %s -a %s"
 200                % (CONST.__getattribute__('openstack_creds'),
 201                   CONST.__getattribute__('INSTALLER_TYPE'),
 202                   CONST.__getattribute__('INSTALLER_IP')))
 203         logger.debug("Executing command: %s" % cmd)
 204         p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
 205         output = p.communicate()[0]
 206         logger.debug("\n%s" % output)
 207         if p.returncode != 0:
 208             raise Exception("Failed to fetch credentials from installer.")
 209     else:
 210         logger.info("RC file provided in %s."
 211                     % CONST.__getattribute__('openstack_creds'))
 212         if os.path.getsize(CONST.__getattribute__('openstack_creds')) == 0:
 213             raise Exception("The file %s is empty." %
 214                             CONST.__getattribute__('openstack_creds'))
 215
 216     logger.info("Sourcing the OpenStack RC file...")
 217     os_utils.source_credentials(CONST.__getattribute__('openstack_creds'))
 218     for key, value in os.environ.iteritems():
 219         if re.search("OS_", key):
 220             if key == 'OS_AUTH_URL':
 221                 CONST.__setattr__('OS_AUTH_URL', value)
 222             elif key == 'OS_USERNAME':
 223                 CONST.__setattr__('OS_USERNAME', value)
 224             elif key == 'OS_TENANT_NAME':
 225                 CONST.__setattr__('OS_TENANT_NAME', value)
 226             elif key == 'OS_PASSWORD':
 227                 CONST.__setattr__('OS_PASSWORD', value)
 228
 229
 230 def patch_config_file():
 231     patch_file(CONFIG_PATCH_PATH)
 232
 233     if pod_arch and pod_arch in arch_filter:
 234         patch_file(CONFIG_AARCH64_PATCH_PATH)
 235
 236
 237 def patch_file(patch_file_path):
 238     logger.debug('Updating file: %s', patch_file_path)
 239     with open(patch_file_path) as f:
 240         patch_file = yaml.safe_load(f)
 241
 242     updated = False
 243     for key in patch_file:
 244         if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
 245             new_functest_yaml = dict(ft_utils.merge_dicts(
 246                 ft_utils.get_functest_yaml(), patch_file[key]))
 247             updated = True
 248
 249     if updated:
 250         os.remove(CONFIG_FUNCTEST_PATH)
 251         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 252             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 253         f.close()
 254
 255
 256 def verify_deployment():
 257     print_separator()
 258     logger.info("Verifying OpenStack services...")
 259     cmd = ("%s/functest/ci/check_os.sh" %
 260            CONST.__getattribute__('dir_repo_functest'))
 261
 262     logger.debug("Executing command: %s" % cmd)
 263     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 264
 265     while p.poll() is None:
 266         line = p.stdout.readline().rstrip()
 267         if "ERROR" in line:
 268             logger.error(line)
 269             raise Exception("Problem while running 'check_os.sh'.")
 270         logger.info(line)
 271
 272
 273 def install_rally():
 274     print_separator()
 275
 276     if pod_arch and pod_arch in arch_filter:
 277         logger.info("Apply aarch64 specific to rally config...")
 278         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 279             rally_patch_conf = f.read()
 280
 281         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 282             print line,
 283             if "cirros|testvm" in line:
 284                 print rally_patch_conf
 285
 286     logger.info("Creating Rally environment...")
 287
 288     cmd = "rally deployment destroy opnfv-rally"
 289     ft_utils.execute_command(cmd, error_msg=(
 290         "Deployment %s does not exist."
 291         % CONST.__getattribute__('rally_deployment_name')),
 292         verbose=False)
 293
 294     rally_conf = os_utils.get_credentials_for_rally()
 295     with open('rally_conf.json', 'w') as fp:
 296         json.dump(rally_conf, fp)
 297     cmd = ("rally deployment create "
 298            "--file=rally_conf.json --name={0}"
 299            .format(CONST.__getattribute__('rally_deployment_name')))
 300     error_msg = "Problem while creating Rally deployment"
 301     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 302
 303     cmd = "rally deployment check"
 304     error_msg = "OpenStack not responding or faulty Rally deployment."
 305     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 306
 307     cmd = "rally deployment list"
 308     ft_utils.execute_command(cmd,
 309                              error_msg=("Problem while listing "
 310                                         "Rally deployment."))
 311
 312     cmd = "rally plugin list | head -5"
 313     ft_utils.execute_command(cmd,
 314                              error_msg=("Problem while showing "
 315                                         "Rally plugins."))
 316
 317
 318 def install_tempest():
 319     logger.info("Installing tempest from existing repo...")
 320     cmd = ("rally verify list-verifiers | "
 321            "grep '{0}' | wc -l".format(
 322                CONST.__getattribute__('tempest_deployment_name')))
 323     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 324     while p.poll() is None:
 325         line = p.stdout.readline().rstrip()
 326         if str(line) == '0':
 327             logger.debug("Tempest %s does not exist" %
 328                          CONST.__getattribute__('tempest_deployment_name'))
 329             cmd = ("rally verify create-verifier --source {0} "
 330                    "--name {1} --type tempest --system-wide"
 331                    .format(CONST.__getattribute__('dir_repo_tempest'),
 332                            CONST.__getattribute__('tempest_deployment_name')))
 333             error_msg = "Problem while installing Tempest."
 334             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 335
 336
 337 def create_flavor():
 338     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 339                                                  '512',
 340                                                  '1',
 341                                                  '1',
 342                                                  public=True)
 343     if flavor_id is None:
 344         raise Exception('Failed to create flavor')
 345
 346
 347 def check_environment():
 348     msg_not_active = "The Functest environment is not installed."
 349     if not os.path.isfile(CONST.__getattribute__('env_active')):
 350         raise Exception(msg_not_active)
 351
 352     with open(CONST.__getattribute__('env_active'), "r") as env_file:
 353         s = env_file.read()
 354         if not re.search("1", s):
 355             raise Exception(msg_not_active)
 356
 357     logger.info("Functest environment is installed.")
 358
 359
 360 def print_deployment_info():
 361     if handler:
 362         logger.info('\n\nDeployment information:\n%s' %
 363                     handler.get_deployment_info())
 364
 365
 366 def main(**kwargs):
 367     try:
 368         if not (kwargs['action'] in actions):
 369             logger.error('Argument not valid.')
 370             return -1
 371         elif kwargs['action'] == "start":
 372             logger.info("######### Preparing Functest environment #########\n")
 373             check_env_variables()
 374             get_deployment_handler()
 375             create_directories()
 376             source_rc_file()
 377             patch_config_file()
 378             verify_deployment()
 379             install_rally()
 380             install_tempest()
 381             create_flavor()
 382             with open(CONST.__getattribute__('env_active'), "w") as env_file:
 383                 env_file.write("1")
 384             check_environment()
 385             print_deployment_info()
 386         elif kwargs['action'] == "check":
 387             check_environment()
 388     except Exception as e:
 389         logger.error(e)
 390         return -1
 391     return 0
 392
 393
 394 if __name__ == '__main__':
 395     logging.config.fileConfig(
 396         CONST.__getattribute__('dir_functest_logging_cfg'))
 397     parser = PrepareEnvParser()
 398     args = parser.parse_args(sys.argv[1:])
 399     sys.exit(main(**args))