apex/deploy.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 import argparse
  13 import git
  14 import json
  15 import logging
  16 import os
  17 import platform
  18 import pprint
  19 import shutil
  20 import sys
  21 import tempfile
  22 import yaml
  23
  24 import apex.virtual.configure_vm as vm_lib
  25 import apex.virtual.utils as virt_utils
  26 import apex.builders.common_builder as c_builder
  27 import apex.builders.overcloud_builder as oc_builder
  28 import apex.builders.undercloud_builder as uc_builder
  29 from apex import DeploySettings
  30 from apex import Inventory
  31 from apex import NetworkEnvironment
  32 from apex import NetworkSettings
  33 from apex.common import utils
  34 from apex.common import constants
  35 from apex.common import parsers
  36 from apex.common.exceptions import ApexDeployException
  37 from apex.network import jumphost
  38 from apex.network import network_data
  39 from apex.undercloud import undercloud as uc_lib
  40 from apex.overcloud import config as oc_cfg
  41 from apex.overcloud import deploy as oc_deploy
  42
  43 APEX_TEMP_DIR = tempfile.mkdtemp(prefix='apex_tmp')
  44 SDN_IMAGE = 'overcloud-full-opendaylight.qcow2'
  45
  46
  47 def deploy_quickstart(args, deploy_settings_file, network_settings_file,
  48                       inventory_file=None):
  49     pass
  50
  51
  52 def validate_cross_settings(deploy_settings, net_settings, inventory):
  53     """
  54     Used to validate compatibility across settings file.
  55     :param deploy_settings: parsed settings for deployment
  56     :param net_settings: parsed settings for network
  57     :param inventory: parsed inventory file
  58     :return: None
  59     """
  60
  61     if deploy_settings['deploy_options']['dataplane'] != 'ovs' and 'tenant' \
  62             not in net_settings.enabled_network_list:
  63         raise ApexDeployException("Setting a DPDK based dataplane requires"
  64                                   "a dedicated NIC for tenant network")
  65
  66     if 'odl_vpp_routing_node' in deploy_settings['deploy_options']:
  67         if deploy_settings['deploy_options']['dataplane'] != 'fdio':
  68             raise ApexDeployException("odl_vpp_routing_node should only be set"
  69                                       "when dataplane is set to fdio")
  70         if deploy_settings['deploy_options'].get('dvr') is True:
  71             raise ApexDeployException("odl_vpp_routing_node should only be set"
  72                                       "when dvr is not enabled")
  73
  74     # TODO(trozet): add more checks here like RAM for ODL, etc
  75     # check if odl_vpp_netvirt is true and vpp is set
  76     # Check if fdio and nosdn:
  77     # tenant_nic_mapping_controller_members" ==
  78     # "$tenant_nic_mapping_compute_members
  79
  80
  81 def build_vms(inventory, network_settings,
  82               template_dir='/usr/share/opnfv-apex'):
  83     """
  84     Creates VMs and configures vbmc and host
  85     :param inventory:
  86     :param network_settings:
  87     :return:
  88     """
  89
  90     for idx, node in enumerate(inventory['nodes']):
  91         name = 'baremetal{}'.format(idx)
  92         volume = name + ".qcow2"
  93         volume_path = os.path.join(constants.LIBVIRT_VOLUME_PATH, volume)
  94         # TODO(trozet): add error checking
  95         vm_lib.create_vm(
  96             name, volume_path,
  97             baremetal_interfaces=network_settings.enabled_network_list,
  98             memory=node['memory'], cpus=node['cpu'],
  99             macs=node['mac'],
 100             template_dir=template_dir)
 101         virt_utils.host_setup({name: node['pm_port']})
 102
 103
 104 def create_deploy_parser():
 105     deploy_parser = argparse.ArgumentParser()
 106     deploy_parser.add_argument('--debug', action='store_true', default=False,
 107                                help="Turn on debug messages")
 108     deploy_parser.add_argument('-l', '--log-file',
 109                                default='./apex_deploy.log',
 110                                dest='log_file', help="Log file to log to")
 111     deploy_parser.add_argument('-d', '--deploy-settings',
 112                                dest='deploy_settings_file',
 113                                required=True,
 114                                help='File which contains Apex deploy settings')
 115     deploy_parser.add_argument('-n', '--network-settings',
 116                                dest='network_settings_file',
 117                                required=True,
 118                                help='File which contains Apex network '
 119                                     'settings')
 120     deploy_parser.add_argument('-i', '--inventory-file',
 121                                dest='inventory_file',
 122                                default=None,
 123                                help='Inventory file which contains POD '
 124                                     'definition')
 125     deploy_parser.add_argument('-e', '--environment-file',
 126                                dest='env_file',
 127                                default='opnfv-environment.yaml',
 128                                help='Provide alternate base env file located '
 129                                     'in deploy_dir')
 130     deploy_parser.add_argument('-v', '--virtual', action='store_true',
 131                                default=False,
 132                                dest='virtual',
 133                                help='Enable virtual deployment')
 134     deploy_parser.add_argument('--interactive', action='store_true',
 135                                default=False,
 136                                help='Enable interactive deployment mode which '
 137                                     'requires user to confirm steps of '
 138                                     'deployment')
 139     deploy_parser.add_argument('--virtual-computes',
 140                                dest='virt_compute_nodes',
 141                                default=1,
 142                                type=int,
 143                                help='Number of Virtual Compute nodes to create'
 144                                     ' and use during deployment (defaults to 1'
 145                                     ' for noha and 2 for ha)')
 146     deploy_parser.add_argument('--virtual-cpus',
 147                                dest='virt_cpus',
 148                                default=4,
 149                                type=int,
 150                                help='Number of CPUs to use per Overcloud VM in'
 151                                     ' a virtual deployment (defaults to 4)')
 152     deploy_parser.add_argument('--virtual-default-ram',
 153                                dest='virt_default_ram',
 154                                default=8,
 155                                type=int,
 156                                help='Amount of default RAM to use per '
 157                                     'Overcloud VM in GB (defaults to 8).')
 158     deploy_parser.add_argument('--virtual-compute-ram',
 159                                dest='virt_compute_ram',
 160                                default=None,
 161                                type=int,
 162                                help='Amount of RAM to use per Overcloud '
 163                                     'Compute VM in GB (defaults to 8). '
 164                                     'Overrides --virtual-default-ram arg for '
 165                                     'computes')
 166     deploy_parser.add_argument('--deploy-dir',
 167                                default='/usr/share/opnfv-apex',
 168                                help='Directory to deploy from which contains '
 169                                     'base config files for deployment')
 170     deploy_parser.add_argument('--image-dir',
 171                                default='/var/opt/opnfv/images',
 172                                help='Directory which contains '
 173                                     'base disk images for deployment')
 174     deploy_parser.add_argument('--lib-dir',
 175                                default='/usr/share/opnfv-apex',
 176                                help='Directory path for apex ansible '
 177                                     'and third party libs')
 178     deploy_parser.add_argument('--quickstart', action='store_true',
 179                                default=False,
 180                                help='Use tripleo-quickstart to deploy')
 181     deploy_parser.add_argument('--upstream', action='store_true',
 182                                default=True,
 183                                help='Force deployment to use upstream '
 184                                     'artifacts. This option is now '
 185                                     'deprecated and only upstream '
 186                                     'deployments are supported.')
 187     deploy_parser.add_argument('--no-fetch', action='store_true',
 188                                default=False,
 189                                help='Ignore fetching latest upstream and '
 190                                     'use what is in cache')
 191     return deploy_parser
 192
 193
 194 def validate_deploy_args(args):
 195     """
 196     Validates arguments for deploy
 197     :param args:
 198     :return: None
 199     """
 200
 201     logging.debug('Validating arguments for deployment')
 202     if args.virtual and args.inventory_file is not None:
 203         logging.error("Virtual enabled but inventory file also given")
 204         raise ApexDeployException('You should not specify an inventory file '
 205                                   'with virtual deployments')
 206     elif args.virtual:
 207         args.inventory_file = os.path.join(APEX_TEMP_DIR,
 208                                            'inventory-virt.yaml')
 209     elif os.path.isfile(args.inventory_file) is False:
 210         logging.error("Specified inventory file does not exist: {}".format(
 211             args.inventory_file))
 212         raise ApexDeployException('Specified inventory file does not exist')
 213
 214     for settings_file in (args.deploy_settings_file,
 215                           args.network_settings_file):
 216         if os.path.isfile(settings_file) is False:
 217             logging.error("Specified settings file does not "
 218                           "exist: {}".format(settings_file))
 219             raise ApexDeployException('Specified settings file does not '
 220                                       'exist: {}'.format(settings_file))
 221
 222
 223 def main():
 224     parser = create_deploy_parser()
 225     args = parser.parse_args(sys.argv[1:])
 226     # FIXME (trozet): this is only needed as a workaround for CI.  Remove
 227     # when CI is changed
 228     if os.getenv('IMAGES', False):
 229         args.image_dir = os.getenv('IMAGES')
 230     if args.debug:
 231         log_level = logging.DEBUG
 232     else:
 233         log_level = logging.INFO
 234     os.makedirs(os.path.dirname(args.log_file), exist_ok=True)
 235     formatter = '%(asctime)s %(levelname)s: %(message)s'
 236     logging.basicConfig(filename=args.log_file,
 237                         format=formatter,
 238                         datefmt='%m/%d/%Y %I:%M:%S %p',
 239                         level=log_level)
 240     console = logging.StreamHandler()
 241     console.setLevel(log_level)
 242     console.setFormatter(logging.Formatter(formatter))
 243     logging.getLogger('').addHandler(console)
 244     utils.install_ansible()
 245     validate_deploy_args(args)
 246     # Parse all settings
 247     deploy_settings = DeploySettings(args.deploy_settings_file)
 248     logging.info("Deploy settings are:\n {}".format(pprint.pformat(
 249         deploy_settings)))
 250     net_settings = NetworkSettings(args.network_settings_file)
 251     logging.info("Network settings are:\n {}".format(pprint.pformat(
 252         net_settings)))
 253     os_version = deploy_settings['deploy_options']['os_version']
 254     net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE)
 255     net_env = NetworkEnvironment(net_settings, net_env_file,
 256                                  os_version=os_version)
 257     net_env_target = os.path.join(APEX_TEMP_DIR, constants.NET_ENV_FILE)
 258     utils.dump_yaml(dict(net_env), net_env_target)
 259
 260     # get global deploy params
 261     ha_enabled = deploy_settings['global_params']['ha_enabled']
 262     introspect = deploy_settings['global_params'].get('introspect', True)
 263
 264     if args.virtual:
 265         if args.virt_compute_ram is None:
 266             compute_ram = args.virt_default_ram
 267         else:
 268             compute_ram = args.virt_compute_ram
 269         if deploy_settings['deploy_options']['sdn_controller'] == \
 270                 'opendaylight' and args.virt_default_ram < 12:
 271             control_ram = 12
 272             logging.warning('RAM per controller is too low.  OpenDaylight '
 273                             'requires at least 12GB per controller.')
 274             logging.info('Increasing RAM per controller to 12GB')
 275         elif args.virt_default_ram < 10:
 276             control_ram = 10
 277             logging.warning('RAM per controller is too low.  nosdn '
 278                             'requires at least 10GB per controller.')
 279             logging.info('Increasing RAM per controller to 10GB')
 280         else:
 281             control_ram = args.virt_default_ram
 282         if ha_enabled and args.virt_compute_nodes < 2:
 283             logging.debug('HA enabled, bumping number of compute nodes to 2')
 284             args.virt_compute_nodes = 2
 285         virt_utils.generate_inventory(args.inventory_file, ha_enabled,
 286                                       num_computes=args.virt_compute_nodes,
 287                                       controller_ram=control_ram * 1024,
 288                                       compute_ram=compute_ram * 1024,
 289                                       vcpus=args.virt_cpus
 290                                       )
 291     inventory = Inventory(args.inventory_file, ha_enabled, args.virtual)
 292
 293     validate_cross_settings(deploy_settings, net_settings, inventory)
 294     ds_opts = deploy_settings['deploy_options']
 295     if args.quickstart:
 296         deploy_settings_file = os.path.join(APEX_TEMP_DIR,
 297                                             'apex_deploy_settings.yaml')
 298         utils.dump_yaml(utils.dict_objects_to_str(deploy_settings),
 299                         deploy_settings_file)
 300         logging.info("File created: {}".format(deploy_settings_file))
 301         network_settings_file = os.path.join(APEX_TEMP_DIR,
 302                                              'apex_network_settings.yaml')
 303         utils.dump_yaml(utils.dict_objects_to_str(net_settings),
 304                         network_settings_file)
 305         logging.info("File created: {}".format(network_settings_file))
 306         deploy_quickstart(args, deploy_settings_file, network_settings_file,
 307                           args.inventory_file)
 308     else:
 309         # TODO (trozet): add logic back from:
 310         # Iedb75994d35b5dc1dd5d5ce1a57277c8f3729dfd (FDIO DVR)
 311         ansible_args = {
 312             'virsh_enabled_networks': net_settings.enabled_network_list
 313         }
 314         utils.run_ansible(ansible_args,
 315                           os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 316                                        'deploy_dependencies.yml'))
 317         uc_external = False
 318         if 'external' in net_settings.enabled_network_list:
 319             uc_external = True
 320         if args.virtual:
 321             # create all overcloud VMs
 322             build_vms(inventory, net_settings, args.deploy_dir)
 323         else:
 324             # Attach interfaces to jumphost for baremetal deployment
 325             jump_networks = ['admin']
 326             if uc_external:
 327                 jump_networks.append('external')
 328             for network in jump_networks:
 329                 if network == 'external':
 330                     # TODO(trozet): enable vlan secondary external networks
 331                     iface = net_settings['networks'][network][0][
 332                         'installer_vm']['members'][0]
 333                 else:
 334                     iface = net_settings['networks'][network]['installer_vm'][
 335                         'members'][0]
 336                 bridge = "br-{}".format(network)
 337                 jumphost.attach_interface_to_ovs(bridge, iface, network)
 338         instackenv_json = os.path.join(APEX_TEMP_DIR, 'instackenv.json')
 339         with open(instackenv_json, 'w') as fh:
 340             json.dump(inventory, fh)
 341
 342         # Create and configure undercloud
 343         if args.debug:
 344             root_pw = constants.DEBUG_OVERCLOUD_PW
 345         else:
 346             root_pw = None
 347
 348         if not args.upstream:
 349             logging.warning("Using upstream is now required for Apex. "
 350                             "Forcing upstream to true")
 351         if os_version == 'master':
 352             branch = 'master'
 353         else:
 354             branch = "stable/{}".format(os_version)
 355
 356         logging.info("Deploying with upstream artifacts for OpenStack "
 357                      "{}".format(os_version))
 358         args.image_dir = os.path.join(args.image_dir, os_version)
 359         upstream_url = constants.UPSTREAM_RDO.replace(
 360             constants.DEFAULT_OS_VERSION, os_version)
 361         upstream_targets = ['overcloud-full.tar', 'undercloud.qcow2']
 362         utils.fetch_upstream_and_unpack(args.image_dir, upstream_url,
 363                                         upstream_targets,
 364                                         fetch=not args.no_fetch)
 365         sdn_image = os.path.join(args.image_dir, 'overcloud-full.qcow2')
 366         # copy undercloud so we don't taint upstream fetch
 367         uc_image = os.path.join(args.image_dir, 'undercloud_mod.qcow2')
 368         uc_fetch_img = os.path.join(args.image_dir, 'undercloud.qcow2')
 369         shutil.copyfile(uc_fetch_img, uc_image)
 370         # prep undercloud with required packages
 371         uc_builder.add_upstream_packages(uc_image)
 372         # add patches from upstream to undercloud and overcloud
 373         logging.info('Adding patches to undercloud')
 374         patches = deploy_settings['global_params']['patches']
 375         c_builder.add_upstream_patches(patches['undercloud'], uc_image,
 376                                        APEX_TEMP_DIR, branch)
 377
 378         # Create/Start Undercloud VM
 379         undercloud = uc_lib.Undercloud(args.image_dir,
 380                                        args.deploy_dir,
 381                                        root_pw=root_pw,
 382                                        external_network=uc_external,
 383                                        image_name=os.path.basename(uc_image),
 384                                        os_version=os_version)
 385         undercloud.start()
 386         undercloud_admin_ip = net_settings['networks'][
 387             constants.ADMIN_NETWORK]['installer_vm']['ip']
 388
 389         if ds_opts['containers']:
 390             tag = constants.DOCKER_TAG
 391         else:
 392             tag = None
 393
 394         # Generate nic templates
 395         for role in 'compute', 'controller':
 396             oc_cfg.create_nic_template(net_settings, deploy_settings, role,
 397                                        args.deploy_dir, APEX_TEMP_DIR)
 398         # Install Undercloud
 399         undercloud.configure(net_settings, deploy_settings,
 400                              os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 401                                           'configure_undercloud.yml'),
 402                              APEX_TEMP_DIR, virtual_oc=args.virtual)
 403
 404         # Prepare overcloud-full.qcow2
 405         logging.info("Preparing Overcloud for deployment...")
 406         if os_version != 'ocata':
 407             net_data_file = os.path.join(APEX_TEMP_DIR, 'network_data.yaml')
 408             net_data = network_data.create_network_data(net_settings,
 409                                                         net_data_file)
 410         else:
 411             net_data = False
 412
 413         # TODO(trozet): Either fix opnfv env or default to use upstream env
 414         if args.env_file == 'opnfv-environment.yaml':
 415             # Override the env_file if it is defaulted to opnfv
 416             # opnfv env file will not work with upstream
 417             args.env_file = 'upstream-environment.yaml'
 418         opnfv_env = os.path.join(args.deploy_dir, args.env_file)
 419         oc_deploy.prep_env(deploy_settings, net_settings, inventory,
 420                            opnfv_env, net_env_target, APEX_TEMP_DIR)
 421         patched_containers = oc_deploy.prep_image(
 422             deploy_settings, net_settings, sdn_image, APEX_TEMP_DIR,
 423             root_pw=root_pw, docker_tag=tag, patches=patches['overcloud'])
 424
 425         oc_deploy.create_deploy_cmd(deploy_settings, net_settings, inventory,
 426                                     APEX_TEMP_DIR, args.virtual,
 427                                     os.path.basename(opnfv_env),
 428                                     net_data=net_data)
 429         # Prepare undercloud with containers
 430         docker_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 431                                        'prepare_overcloud_containers.yml')
 432         if ds_opts['containers']:
 433             ceph_version = constants.CEPH_VERSION_MAP[ds_opts['os_version']]
 434             ceph_docker_image = "ceph/daemon:tag-build-master-" \
 435                                 "{}-centos-7".format(ceph_version)
 436             logging.info("Preparing Undercloud with Docker containers")
 437             if patched_containers:
 438                 oc_builder.archive_docker_patches(APEX_TEMP_DIR)
 439             container_vars = dict()
 440             container_vars['apex_temp_dir'] = APEX_TEMP_DIR
 441             container_vars['patched_docker_services'] = list(
 442                 patched_containers)
 443             container_vars['container_tag'] = constants.DOCKER_TAG
 444             container_vars['stackrc'] = 'source /home/stack/stackrc'
 445             container_vars['sdn'] = ds_opts['sdn_controller']
 446             container_vars['undercloud_ip'] = undercloud_admin_ip
 447             container_vars['os_version'] = os_version
 448             container_vars['ceph_docker_image'] = ceph_docker_image
 449             container_vars['sdn_env_file'] = \
 450                 oc_deploy.get_docker_sdn_file(ds_opts)
 451             try:
 452                 utils.run_ansible(container_vars, docker_playbook,
 453                                   host=undercloud.ip, user='stack',
 454                                   tmp_dir=APEX_TEMP_DIR)
 455                 logging.info("Container preparation complete")
 456             except Exception:
 457                 logging.error("Unable to complete container prep on "
 458                               "Undercloud")
 459                 os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
 460                 raise
 461
 462         deploy_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 463                                        'deploy_overcloud.yml')
 464         virt_env = 'virtual-environment.yaml'
 465         bm_env = 'baremetal-environment.yaml'
 466         k8s_env = 'kubernetes-environment.yaml'
 467         for p_env in virt_env, bm_env, k8s_env:
 468             shutil.copyfile(os.path.join(args.deploy_dir, p_env),
 469                             os.path.join(APEX_TEMP_DIR, p_env))
 470
 471         # Start Overcloud Deployment
 472         logging.info("Executing Overcloud Deployment...")
 473         deploy_vars = dict()
 474         deploy_vars['virtual'] = args.virtual
 475         deploy_vars['debug'] = args.debug
 476         deploy_vars['aarch64'] = platform.machine() == 'aarch64'
 477         deploy_vars['introspect'] = not (args.virtual or
 478                                          deploy_vars['aarch64'] or
 479                                          not introspect)
 480         deploy_vars['dns_server_args'] = ''
 481         deploy_vars['apex_temp_dir'] = APEX_TEMP_DIR
 482         deploy_vars['apex_env_file'] = os.path.basename(opnfv_env)
 483         deploy_vars['stackrc'] = 'source /home/stack/stackrc'
 484         deploy_vars['overcloudrc'] = 'source /home/stack/overcloudrc'
 485         deploy_vars['undercloud_ip'] = undercloud_admin_ip
 486         deploy_vars['ha_enabled'] = ha_enabled
 487         deploy_vars['os_version'] = os_version
 488         deploy_vars['http_proxy'] = net_settings.get('http_proxy', '')
 489         deploy_vars['https_proxy'] = net_settings.get('https_proxy', '')
 490         deploy_vars['vim'] = ds_opts['vim']
 491         for dns_server in net_settings['dns_servers']:
 492             deploy_vars['dns_server_args'] += " --dns-nameserver {}".format(
 493                 dns_server)
 494         try:
 495             utils.run_ansible(deploy_vars, deploy_playbook, host=undercloud.ip,
 496                               user='stack', tmp_dir=APEX_TEMP_DIR)
 497             logging.info("Overcloud deployment complete")
 498         except Exception:
 499             logging.error("Deployment Failed.  Please check deploy log as "
 500                           "well as mistral logs in "
 501                           "{}".format(os.path.join(APEX_TEMP_DIR,
 502                                                    'mistral_logs.tar.gz')))
 503             raise
 504         finally:
 505             os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
 506
 507         # Post install
 508         logging.info("Executing post deploy configuration")
 509         jumphost.configure_bridges(net_settings)
 510         nova_output = os.path.join(APEX_TEMP_DIR, 'nova_output')
 511         deploy_vars['overcloud_nodes'] = parsers.parse_nova_output(
 512             nova_output)
 513         deploy_vars['SSH_OPTIONS'] = '-o StrictHostKeyChecking=no -o ' \
 514                                      'GlobalKnownHostsFile=/dev/null -o ' \
 515                                      'UserKnownHostsFile=/dev/null -o ' \
 516                                      'LogLevel=error'
 517         deploy_vars['external_network_cmds'] = \
 518             oc_deploy.external_network_cmds(net_settings, deploy_settings)
 519         # TODO(trozet): just parse all ds_opts as deploy vars one time
 520         deploy_vars['gluon'] = ds_opts['gluon']
 521         deploy_vars['sdn'] = ds_opts['sdn_controller']
 522         for dep_option in 'yardstick', 'dovetail', 'vsperf':
 523             if dep_option in ds_opts:
 524                 deploy_vars[dep_option] = ds_opts[dep_option]
 525             else:
 526                 deploy_vars[dep_option] = False
 527         deploy_vars['dataplane'] = ds_opts['dataplane']
 528         overcloudrc = os.path.join(APEX_TEMP_DIR, 'overcloudrc')
 529         if ds_opts['congress']:
 530             deploy_vars['congress_datasources'] = \
 531                 oc_deploy.create_congress_cmds(overcloudrc)
 532             deploy_vars['congress'] = True
 533         else:
 534             deploy_vars['congress'] = False
 535         deploy_vars['calipso'] = ds_opts.get('calipso', False)
 536         deploy_vars['calipso_ip'] = undercloud_admin_ip
 537         # overcloudrc.v3 removed and set as default in queens and later
 538         if os_version == 'pike':
 539             deploy_vars['overcloudrc_files'] = ['overcloudrc',
 540                                                 'overcloudrc.v3']
 541         else:
 542             deploy_vars['overcloudrc_files'] = ['overcloudrc']
 543
 544         post_undercloud = os.path.join(args.lib_dir,
 545                                        constants.ANSIBLE_PATH,
 546                                        'post_deploy_undercloud.yml')
 547         logging.info("Executing post deploy configuration undercloud "
 548                      "playbook")
 549         try:
 550             utils.run_ansible(deploy_vars, post_undercloud,
 551                               host=undercloud.ip, user='stack',
 552                               tmp_dir=APEX_TEMP_DIR)
 553             logging.info("Post Deploy Undercloud Configuration Complete")
 554         except Exception:
 555             logging.error("Post Deploy Undercloud Configuration failed.  "
 556                           "Please check log")
 557             raise
 558
 559         # Deploy kubernetes if enabled
 560         # (TODO)zshi move handling of kubernetes deployment
 561         # to its own deployment class
 562         if deploy_vars['vim'] == 'k8s':
 563             # clone kubespray repo
 564             git.Repo.clone_from(constants.KUBESPRAY_URL,
 565                                 os.path.join(APEX_TEMP_DIR, 'kubespray'))
 566             shutil.copytree(
 567                 os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
 568                              'sample'),
 569                 os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
 570                              'apex'))
 571             k8s_node_inventory = {
 572                 'all':
 573                     {'hosts': {},
 574                      'children': {
 575                          'k8s-cluster': {
 576                              'children': {
 577                                  'kube-master': {
 578                                      'hosts': {}
 579                                  },
 580                                  'kube-node': {
 581                                      'hosts': {}
 582                                  }
 583                              }
 584                          },
 585                          'etcd': {
 586                              'hosts': {}
 587                          }
 588                     }
 589                     }
 590             }
 591             for node, ip in deploy_vars['overcloud_nodes'].items():
 592                 k8s_node_inventory['all']['hosts'][node] = {
 593                     'ansible_become': True,
 594                     'ansible_ssh_host': ip,
 595                     'ansible_become_user': 'root',
 596                     'ip': ip
 597                 }
 598                 if 'controller' in node:
 599                     k8s_node_inventory['all']['children']['k8s-cluster'][
 600                         'children']['kube-master']['hosts'][node] = None
 601                     k8s_node_inventory['all']['children']['etcd'][
 602                         'hosts'][node] = None
 603                 elif 'compute' in node:
 604                     k8s_node_inventory['all']['children']['k8s-cluster'][
 605                         'children']['kube-node']['hosts'][node] = None
 606
 607             kubespray_dir = os.path.join(APEX_TEMP_DIR, 'kubespray')
 608             with open(os.path.join(kubespray_dir, 'inventory', 'apex',
 609                                    'apex.yaml'), 'w') as invfile:
 610                 yaml.dump(k8s_node_inventory, invfile,
 611                           default_flow_style=False)
 612             k8s_deploy_vars = {}
 613             # Add kubespray ansible control variables in k8s_deploy_vars,
 614             # example: 'kube_network_plugin': 'flannel'
 615             k8s_deploy = os.path.join(kubespray_dir, 'cluster.yml')
 616             k8s_deploy_inv_file = os.path.join(kubespray_dir, 'inventory',
 617                                                'apex', 'apex.yaml')
 618
 619             k8s_remove_pkgs = os.path.join(args.lib_dir,
 620                                            constants.ANSIBLE_PATH,
 621                                            'k8s_remove_pkgs.yml')
 622             try:
 623                 logging.debug("Removing any existing overcloud docker "
 624                               "packages")
 625                 utils.run_ansible(k8s_deploy_vars, k8s_remove_pkgs,
 626                                   host=k8s_deploy_inv_file,
 627                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 628                 logging.info("k8s Deploy Remove Existing Docker Related "
 629                              "Packages Complete")
 630             except Exception:
 631                 logging.error("k8s Deploy Remove Existing Docker Related "
 632                               "Packages failed. Please check log")
 633                 raise
 634
 635             try:
 636                 utils.run_ansible(k8s_deploy_vars, k8s_deploy,
 637                                   host=k8s_deploy_inv_file,
 638                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 639                 logging.info("k8s Deploy Overcloud Configuration Complete")
 640             except Exception:
 641                 logging.error("k8s Deploy Overcloud Configuration failed."
 642                               "Please check log")
 643                 raise
 644
 645         # Post deploy overcloud node configuration
 646         # TODO(trozet): just parse all ds_opts as deploy vars one time
 647         deploy_vars['sfc'] = ds_opts['sfc']
 648         deploy_vars['vpn'] = ds_opts['vpn']
 649         deploy_vars['l2gw'] = ds_opts.get('l2gw')
 650         deploy_vars['sriov'] = ds_opts.get('sriov')
 651         deploy_vars['tacker'] = ds_opts.get('tacker')
 652         # TODO(trozet): pull all logs and store in tmp dir in overcloud
 653         # playbook
 654         post_overcloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 655                                       'post_deploy_overcloud.yml')
 656         # Run per overcloud node
 657         for node, ip in deploy_vars['overcloud_nodes'].items():
 658             logging.info("Executing Post deploy overcloud playbook on "
 659                          "node {}".format(node))
 660             try:
 661                 utils.run_ansible(deploy_vars, post_overcloud, host=ip,
 662                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 663                 logging.info("Post Deploy Overcloud Configuration Complete "
 664                              "for node {}".format(node))
 665             except Exception:
 666                 logging.error("Post Deploy Overcloud Configuration failed "
 667                               "for node {}. Please check log".format(node))
 668                 raise
 669         logging.info("Apex deployment complete")
 670         logging.info("Undercloud IP: {}, please connect by doing "
 671                      "'opnfv-util undercloud'".format(undercloud.ip))
 672         # TODO(trozet): add logging here showing controller VIP and horizon url
 673
 674
 675 if __name__ == '__main__':
 676     main()