functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import re
  15 import subprocess
  16 import sys
  17 import fileinput
  18
  19 import yaml
  20
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23 from functest.utils.constants import CONST
  24
  25 from opnfv.utils import constants as opnfv_constants
  26 from opnfv.deployment import factory
  27
  28 actions = ['start', 'check']
  29
  30 """ logging configuration """
  31 logger = logging.getLogger('functest.ci.prepare_env')
  32 handler = None
  33 # set the architecture to default
  34 pod_arch = None
  35 arch_filter = ['aarch64']
  36
  37 CONFIG_FUNCTEST_PATH = CONST.__getattribute__('CONFIG_FUNCTEST_YAML')
  38 CONFIG_PATCH_PATH = os.path.join(os.path.dirname(
  39     CONFIG_FUNCTEST_PATH), "config_patch.yaml")
  40 CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  41     CONFIG_FUNCTEST_PATH), "config_aarch64_patch.yaml")
  42 RALLY_CONF_PATH = os.path.join("/etc/rally/rally.conf")
  43 RALLY_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  44     CONFIG_FUNCTEST_PATH), "rally_aarch64_patch.conf")
  45
  46
  47 class PrepareEnvParser(object):
  48
  49     def __init__(self):
  50         self.parser = argparse.ArgumentParser()
  51         self.parser.add_argument("action", help="Possible actions are: "
  52                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  53                                  choices=actions)
  54         self.parser.add_argument("-d", "--debug", help="Debug mode",
  55                                  action="store_true")
  56
  57     def parse_args(self, argv=[]):
  58         return vars(self.parser.parse_args(argv))
  59
  60
  61 def print_separator():
  62     logger.info("==============================================")
  63
  64
  65 def check_env_variables():
  66     print_separator()
  67     logger.info("Checking environment variables...")
  68
  69     if CONST.__getattribute__('INSTALLER_TYPE') is None:
  70         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  71         CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  72     else:
  73         if (CONST.__getattribute__('INSTALLER_TYPE') not in
  74                 opnfv_constants.INSTALLERS):
  75             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  76                            "Available OPNFV Installers are : %s. "
  77                            "Setting INSTALLER_TYPE=undefined."
  78                            % (CONST.__getattribute__('INSTALLER_TYPE'),
  79                               opnfv_constants.INSTALLERS))
  80             CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  81         else:
  82             logger.info("    INSTALLER_TYPE=%s"
  83                         % CONST.__getattribute__('INSTALLER_TYPE'))
  84
  85     if CONST.__getattribute__('INSTALLER_IP') is None:
  86         logger.warning("The env variable 'INSTALLER_IP' is not defined. "
  87                        "It is needed to fetch the OpenStack credentials. "
  88                        "If the credentials are not provided to the "
  89                        "container as a volume, please add this env variable "
  90                        "to the 'docker run' command.")
  91     else:
  92         logger.info("    INSTALLER_IP=%s" %
  93                     CONST.__getattribute__('INSTALLER_IP'))
  94
  95     if CONST.__getattribute__('DEPLOY_SCENARIO') is None:
  96         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  97                        "Setting CI_SCENARIO=undefined.")
  98         CONST.__setattr__('DEPLOY_SCENARIO', 'undefined')
  99     else:
 100         logger.info("    DEPLOY_SCENARIO=%s"
 101                     % CONST.__getattribute__('DEPLOY_SCENARIO'))
 102     if CONST.__getattribute__('CI_DEBUG'):
 103         logger.info("    CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG'))
 104
 105     if CONST.__getattribute__('NODE_NAME'):
 106         logger.info("    NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME'))
 107
 108     if CONST.__getattribute__('BUILD_TAG'):
 109         logger.info("    BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG'))
 110
 111     if CONST.__getattribute__('IS_CI_RUN'):
 112         logger.info("    IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN'))
 113
 114
 115 def get_deployment_handler():
 116     global handler
 117     global pod_arch
 118
 119     installer_params_yaml = os.path.join(
 120         CONST.__getattribute__('dir_repo_functest'),
 121         'functest/ci/installer_params.yaml')
 122     if (CONST.__getattribute__('INSTALLER_IP') and
 123         CONST.__getattribute__('INSTALLER_TYPE') and
 124             CONST.__getattribute__('INSTALLER_TYPE') in
 125             opnfv_constants.INSTALLERS):
 126         try:
 127             installer_params = ft_utils.get_parameter_from_yaml(
 128                 CONST.__getattribute__('INSTALLER_TYPE'),
 129                 installer_params_yaml)
 130         except ValueError as e:
 131             logger.debug('Printing deployment info is not supported for %s' %
 132                          CONST.__getattribute__('INSTALLER_TYPE'))
 133             logger.debug(e)
 134         else:
 135             user = installer_params.get('user', None)
 136             password = installer_params.get('password', None)
 137             pkey = installer_params.get('pkey', None)
 138             try:
 139                 handler = factory.Factory.get_handler(
 140                     installer=CONST.__getattribute__('INSTALLER_TYPE'),
 141                     installer_ip=CONST.__getattribute__('INSTALLER_IP'),
 142                     installer_user=user,
 143                     installer_pwd=password,
 144                     pkey_file=pkey)
 145                 if handler:
 146                     pod_arch = handler.get_arch()
 147             except Exception as e:
 148                 logger.debug("Cannot get deployment information. %s" % e)
 149
 150
 151 def create_directories():
 152     print_separator()
 153     logger.info("Creating needed directories...")
 154     if not os.path.exists(CONST.__getattribute__('dir_functest_conf')):
 155         os.makedirs(CONST.__getattribute__('dir_functest_conf'))
 156         logger.info("    %s created." %
 157                     CONST.__getattribute__('dir_functest_conf'))
 158     else:
 159         logger.debug("   %s already exists." %
 160                      CONST.__getattribute__('dir_functest_conf'))
 161
 162     if not os.path.exists(CONST.__getattribute__('dir_functest_data')):
 163         os.makedirs(CONST.__getattribute__('dir_functest_data'))
 164         logger.info("    %s created." %
 165                     CONST.__getattribute__('dir_functest_data'))
 166     else:
 167         logger.debug("   %s already exists." %
 168                      CONST.__getattribute__('dir_functest_data'))
 169     if not os.path.exists(CONST.__getattribute__('dir_functest_images')):
 170         os.makedirs(CONST.__getattribute__('dir_functest_images'))
 171         logger.info("    %s created." %
 172                     CONST.__getattribute__('dir_functest_images'))
 173     else:
 174         logger.debug("   %s already exists." %
 175                      CONST.__getattribute__('dir_functest_images'))
 176
 177
 178 def source_rc_file():
 179     print_separator()
 180     logger.info("Fetching RC file...")
 181
 182     if CONST.__getattribute__('openstack_creds') is None:
 183         logger.warning("The environment variable 'creds' must be set and"
 184                        "pointing to the local RC file. Using default: "
 185                        "/home/opnfv/functest/conf/openstack.creds ...")
 186         os.path.join(
 187             CONST.__getattribute__('dir_functest_conf'), 'openstack.creds')
 188
 189     if not os.path.isfile(CONST.__getattribute__('openstack_creds')):
 190         logger.info("RC file not provided. "
 191                     "Fetching it from the installer...")
 192         if CONST.__getattribute__('INSTALLER_IP')is None:
 193             logger.error("The env variable 'INSTALLER_IP' must be provided in"
 194                          " order to fetch the credentials from the installer.")
 195             raise Exception("Missing CI_INSTALLER_IP.")
 196         if (CONST.__getattribute__('INSTALLER_TYPE') not in
 197                 opnfv_constants.INSTALLERS):
 198             logger.error("Cannot fetch credentials. INSTALLER_TYPE=%s is "
 199                          "not a valid OPNFV installer. Available "
 200                          "installers are : %s." %
 201                          (CONST.__getattribute__('INSTALLER_TYPE'),
 202                           opnfv_constants.INSTALLERS))
 203             raise Exception("Wrong INSTALLER_TYPE.")
 204
 205         cmd = ("/home/opnfv/repos/releng/utils/fetch_os_creds.sh "
 206                "-d %s -i %s -a %s"
 207                % (CONST.__getattribute__('openstack_creds'),
 208                   CONST.__getattribute__('INSTALLER_TYPE'),
 209                   CONST.__getattribute__('INSTALLER_IP')))
 210         logger.debug("Executing command: %s" % cmd)
 211         p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
 212         output = p.communicate()[0]
 213         logger.debug("\n%s" % output)
 214         if p.returncode != 0:
 215             raise Exception("Failed to fetch credentials from installer.")
 216     else:
 217         logger.info("RC file provided in %s."
 218                     % CONST.__getattribute__('openstack_creds'))
 219         if os.path.getsize(CONST.__getattribute__('openstack_creds')) == 0:
 220             raise Exception("The file %s is empty." %
 221                             CONST.__getattribute__('openstack_creds'))
 222
 223     logger.info("Sourcing the OpenStack RC file...")
 224     os_utils.source_credentials(CONST.__getattribute__('openstack_creds'))
 225     for key, value in os.environ.iteritems():
 226         if re.search("OS_", key):
 227             if key == 'OS_AUTH_URL':
 228                 CONST.__setattr__('OS_AUTH_URL', value)
 229             elif key == 'OS_USERNAME':
 230                 CONST.__setattr__('OS_USERNAME', value)
 231             elif key == 'OS_TENANT_NAME':
 232                 CONST.__setattr__('OS_TENANT_NAME', value)
 233             elif key == 'OS_PASSWORD':
 234                 CONST.__setattr__('OS_PASSWORD', value)
 235
 236
 237 def patch_config_file():
 238     patch_file(CONFIG_PATCH_PATH)
 239
 240     if pod_arch and pod_arch in arch_filter:
 241         patch_file(CONFIG_AARCH64_PATCH_PATH)
 242
 243
 244 def patch_file(patch_file_path):
 245     logger.debug('Updating file: %s', patch_file_path)
 246     with open(patch_file_path) as f:
 247         patch_file = yaml.safe_load(f)
 248
 249     updated = False
 250     for key in patch_file:
 251         if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
 252             new_functest_yaml = dict(ft_utils.merge_dicts(
 253                 ft_utils.get_functest_yaml(), patch_file[key]))
 254             updated = True
 255
 256     if updated:
 257         os.remove(CONFIG_FUNCTEST_PATH)
 258         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 259             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 260         f.close()
 261
 262
 263 def verify_deployment():
 264     print_separator()
 265     logger.info("Verifying OpenStack services...")
 266     cmd = ("%s/functest/ci/check_os.sh" %
 267            CONST.__getattribute__('dir_repo_functest'))
 268
 269     logger.debug("Executing command: %s" % cmd)
 270     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 271
 272     while p.poll() is None:
 273         line = p.stdout.readline().rstrip()
 274         if "ERROR" in line:
 275             logger.error(line)
 276             raise Exception("Problem while running 'check_os.sh'.")
 277         logger.info(line)
 278
 279
 280 def install_rally():
 281     print_separator()
 282
 283     if pod_arch and pod_arch in arch_filter:
 284         logger.info("Apply aarch64 specific to rally config...")
 285         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 286             rally_patch_conf = f.read()
 287
 288         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 289             print line,
 290             if "cirros|testvm" in line:
 291                 print rally_patch_conf
 292
 293     logger.info("Creating Rally environment...")
 294
 295     cmd = "rally deployment destroy opnfv-rally"
 296     ft_utils.execute_command(cmd, error_msg=(
 297         "Deployment %s does not exist."
 298         % CONST.__getattribute__('rally_deployment_name')),
 299         verbose=False)
 300
 301     rally_conf = os_utils.get_credentials_for_rally()
 302     with open('rally_conf.json', 'w') as fp:
 303         json.dump(rally_conf, fp)
 304     cmd = ("rally deployment create "
 305            "--file=rally_conf.json --name={0}"
 306            .format(CONST.__getattribute__('rally_deployment_name')))
 307     error_msg = "Problem while creating Rally deployment"
 308     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 309
 310     cmd = "rally deployment check"
 311     error_msg = "OpenStack not responding or faulty Rally deployment."
 312     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 313
 314     cmd = "rally deployment list"
 315     ft_utils.execute_command(cmd,
 316                              error_msg=("Problem while listing "
 317                                         "Rally deployment."))
 318
 319     cmd = "rally plugin list | head -5"
 320     ft_utils.execute_command(cmd,
 321                              error_msg=("Problem while showing "
 322                                         "Rally plugins."))
 323
 324
 325 def install_tempest():
 326     logger.info("Installing tempest from existing repo...")
 327     cmd = ("rally verify list-verifiers | "
 328            "grep '{0}' | wc -l".format(
 329                CONST.__getattribute__('tempest_deployment_name')))
 330     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 331     while p.poll() is None:
 332         line = p.stdout.readline().rstrip()
 333         if str(line) == '0':
 334             logger.debug("Tempest %s does not exist" %
 335                          CONST.__getattribute__('tempest_deployment_name'))
 336             cmd = ("rally verify create-verifier --source {0} "
 337                    "--name {1} --type tempest --system-wide"
 338                    .format(CONST.__getattribute__('dir_repo_tempest'),
 339                            CONST.__getattribute__('tempest_deployment_name')))
 340             error_msg = "Problem while installing Tempest."
 341             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 342
 343
 344 def create_flavor():
 345     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 346                                                  '512',
 347                                                  '1',
 348                                                  '1',
 349                                                  public=True)
 350     if flavor_id is None:
 351         raise Exception('Failed to create flavor')
 352
 353
 354 def check_environment():
 355     msg_not_active = "The Functest environment is not installed."
 356     if not os.path.isfile(CONST.__getattribute__('env_active')):
 357         raise Exception(msg_not_active)
 358
 359     with open(CONST.__getattribute__('env_active'), "r") as env_file:
 360         s = env_file.read()
 361         if not re.search("1", s):
 362             raise Exception(msg_not_active)
 363
 364     logger.info("Functest environment is installed.")
 365
 366
 367 def print_deployment_info():
 368     if handler:
 369         logger.info('\n\nDeployment information:\n%s' %
 370                     handler.get_deployment_info())
 371
 372
 373 def main(**kwargs):
 374     try:
 375         if not (kwargs['action'] in actions):
 376             logger.error('Argument not valid.')
 377             return -1
 378         elif kwargs['action'] == "start":
 379             logger.info("######### Preparing Functest environment #########\n")
 380             check_env_variables()
 381             get_deployment_handler()
 382             create_directories()
 383             source_rc_file()
 384             patch_config_file()
 385             verify_deployment()
 386             install_rally()
 387             install_tempest()
 388             create_flavor()
 389             with open(CONST.__getattribute__('env_active'), "w") as env_file:
 390                 env_file.write("1")
 391             check_environment()
 392             print_deployment_info()
 393         elif kwargs['action'] == "check":
 394             check_environment()
 395     except Exception as e:
 396         logger.error(e)
 397         return -1
 398     return 0
 399
 400
 401 if __name__ == '__main__':
 402     logging.config.fileConfig(
 403         CONST.__getattribute__('dir_functest_logging_cfg'))
 404     parser = PrepareEnvParser()
 405     args = parser.parse_args(sys.argv[1:])
 406     sys.exit(main(**args))