apex/deploy.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 import argparse
  13 import git
  14 import json
  15 import logging
  16 import os
  17 import platform
  18 import pprint
  19 import shutil
  20 import sys
  21 import tempfile
  22 import yaml
  23
  24 import apex.virtual.configure_vm as vm_lib
  25 import apex.virtual.utils as virt_utils
  26 import apex.builders.common_builder as c_builder
  27 import apex.builders.overcloud_builder as oc_builder
  28 import apex.builders.undercloud_builder as uc_builder
  29 from apex import DeploySettings
  30 from apex import Inventory
  31 from apex import NetworkEnvironment
  32 from apex import NetworkSettings
  33 from apex.deployment.snapshot import SnapshotDeployment
  34 from apex.common import utils
  35 from apex.common import constants
  36 from apex.common import parsers
  37 from apex.common.exceptions import ApexDeployException
  38 from apex.deployment.tripleo import ApexDeployment
  39 from apex.network import jumphost
  40 from apex.network import network_data
  41 from apex.undercloud import undercloud as uc_lib
  42 from apex.overcloud import config as oc_cfg
  43 from apex.overcloud import deploy as oc_deploy
  44
  45 APEX_TEMP_DIR = tempfile.mkdtemp(prefix='apex_tmp')
  46 SDN_IMAGE = 'overcloud-full-opendaylight.qcow2'
  47
  48
  49 def validate_cross_settings(deploy_settings, net_settings, inventory):
  50     """
  51     Used to validate compatibility across settings file.
  52     :param deploy_settings: parsed settings for deployment
  53     :param net_settings: parsed settings for network
  54     :param inventory: parsed inventory file
  55     :return: None
  56     """
  57
  58     if deploy_settings['deploy_options']['dataplane'] != 'ovs' and 'tenant' \
  59             not in net_settings.enabled_network_list:
  60         raise ApexDeployException("Setting a DPDK based dataplane requires"
  61                                   "a dedicated NIC for tenant network")
  62
  63     if 'odl_vpp_routing_node' in deploy_settings['deploy_options']:
  64         if deploy_settings['deploy_options']['dataplane'] != 'fdio':
  65             raise ApexDeployException("odl_vpp_routing_node should only be set"
  66                                       "when dataplane is set to fdio")
  67         if deploy_settings['deploy_options'].get('dvr') is True:
  68             raise ApexDeployException("odl_vpp_routing_node should only be set"
  69                                       "when dvr is not enabled")
  70
  71     # TODO(trozet): add more checks here like RAM for ODL, etc
  72     # check if odl_vpp_netvirt is true and vpp is set
  73     # Check if fdio and nosdn:
  74     # tenant_nic_mapping_controller_members" ==
  75     # "$tenant_nic_mapping_compute_members
  76
  77
  78 def build_vms(inventory, network_settings,
  79               template_dir='/usr/share/opnfv-apex'):
  80     """
  81     Creates VMs and configures vbmc and host
  82     :param inventory:
  83     :param network_settings:
  84     :return:
  85     """
  86
  87     for idx, node in enumerate(inventory['nodes']):
  88         name = 'baremetal{}'.format(idx)
  89         volume = name + ".qcow2"
  90         volume_path = os.path.join(constants.LIBVIRT_VOLUME_PATH, volume)
  91         # TODO(trozet): add error checking
  92         vm_lib.create_vm(
  93             name, volume_path,
  94             baremetal_interfaces=network_settings.enabled_network_list,
  95             memory=node['memory'], cpus=node['cpu'],
  96             macs=node['mac'],
  97             template_dir=template_dir)
  98         virt_utils.host_setup({name: node['pm_port']})
  99
 100
 101 def create_deploy_parser():
 102     deploy_parser = argparse.ArgumentParser()
 103     deploy_parser.add_argument('--debug', action='store_true', default=False,
 104                                help="Turn on debug messages")
 105     deploy_parser.add_argument('-l', '--log-file',
 106                                default='./apex_deploy.log',
 107                                dest='log_file', help="Log file to log to")
 108     deploy_parser.add_argument('-d', '--deploy-settings',
 109                                dest='deploy_settings_file',
 110                                required=True,
 111                                help='File which contains Apex deploy settings')
 112     deploy_parser.add_argument('-n', '--network-settings',
 113                                dest='network_settings_file',
 114                                required=False,
 115                                help='File which contains Apex network '
 116                                     'settings')
 117     deploy_parser.add_argument('-i', '--inventory-file',
 118                                dest='inventory_file',
 119                                default=None,
 120                                help='Inventory file which contains POD '
 121                                     'definition')
 122     deploy_parser.add_argument('-e', '--environment-file',
 123                                dest='env_file',
 124                                default='opnfv-environment.yaml',
 125                                help='Provide alternate base env file located '
 126                                     'in deploy_dir')
 127     deploy_parser.add_argument('-v', '--virtual', action='store_true',
 128                                default=False,
 129                                dest='virtual',
 130                                help='Enable virtual deployment')
 131     deploy_parser.add_argument('--interactive', action='store_true',
 132                                default=False,
 133                                help='Enable interactive deployment mode which '
 134                                     'requires user to confirm steps of '
 135                                     'deployment')
 136     deploy_parser.add_argument('--virtual-computes',
 137                                dest='virt_compute_nodes',
 138                                default=1,
 139                                type=int,
 140                                help='Number of Virtual Compute nodes to create'
 141                                     ' and use during deployment (defaults to 1'
 142                                     ' for noha and 2 for ha)')
 143     deploy_parser.add_argument('--virtual-cpus',
 144                                dest='virt_cpus',
 145                                default=4,
 146                                type=int,
 147                                help='Number of CPUs to use per Overcloud VM in'
 148                                     ' a virtual deployment (defaults to 4)')
 149     deploy_parser.add_argument('--virtual-default-ram',
 150                                dest='virt_default_ram',
 151                                default=8,
 152                                type=int,
 153                                help='Amount of default RAM to use per '
 154                                     'Overcloud VM in GB (defaults to 8).')
 155     deploy_parser.add_argument('--virtual-compute-ram',
 156                                dest='virt_compute_ram',
 157                                default=None,
 158                                type=int,
 159                                help='Amount of RAM to use per Overcloud '
 160                                     'Compute VM in GB (defaults to 8). '
 161                                     'Overrides --virtual-default-ram arg for '
 162                                     'computes')
 163     deploy_parser.add_argument('--deploy-dir',
 164                                default='/usr/share/opnfv-apex',
 165                                help='Directory to deploy from which contains '
 166                                     'base config files for deployment')
 167     deploy_parser.add_argument('--image-dir',
 168                                default='/var/opt/opnfv/images',
 169                                help='Directory which contains '
 170                                     'base disk images for deployment')
 171     deploy_parser.add_argument('--lib-dir',
 172                                default='/usr/share/opnfv-apex',
 173                                help='Directory path for apex ansible '
 174                                     'and third party libs')
 175     deploy_parser.add_argument('-s', '--snapshot', action='store_true',
 176                                default=False,
 177                                help='Use snapshots for deployment')
 178     deploy_parser.add_argument('--snap-cache', dest='snap_cache',
 179                                default="{}/snap_cache".format(
 180                                    os.path.expanduser('~')),
 181                                help='Local directory to cache snapshot '
 182                                     'artifacts. Defaults to $HOME/snap_cache')
 183     deploy_parser.add_argument('--upstream', action='store_true',
 184                                default=True,
 185                                help='Force deployment to use upstream '
 186                                     'artifacts. This option is now '
 187                                     'deprecated and only upstream '
 188                                     'deployments are supported.')
 189     deploy_parser.add_argument('--no-fetch', action='store_true',
 190                                default=False,
 191                                help='Ignore fetching latest upstream and '
 192                                     'use what is in cache')
 193     deploy_parser.add_argument('-p', '--patches',
 194                                default='/etc/opnfv-apex/common-patches.yaml',
 195                                dest='patches_file',
 196                                help='File to include for common patches '
 197                                     'which apply to all deployment scenarios')
 198     return deploy_parser
 199
 200
 201 def validate_deploy_args(args):
 202     """
 203     Validates arguments for deploy
 204     :param args:
 205     :return: None
 206     """
 207
 208     logging.debug('Validating arguments for deployment')
 209     if args.snapshot:
 210         logging.debug('Skipping inventory validation as it is not applicable'
 211                       'to snapshot deployments')
 212     elif args.virtual and args.inventory_file is not None:
 213         logging.error("Virtual enabled but inventory file also given")
 214         raise ApexDeployException('You should not specify an inventory file '
 215                                   'with virtual deployments')
 216     elif args.virtual:
 217         args.inventory_file = os.path.join(APEX_TEMP_DIR,
 218                                            'inventory-virt.yaml')
 219     elif not os.path.isfile(args.inventory_file):
 220         logging.error("Specified inventory file does not exist: {}".format(
 221             args.inventory_file))
 222         raise ApexDeployException('Specified inventory file does not exist')
 223
 224     for settings_file in (args.deploy_settings_file,
 225                           args.network_settings_file):
 226         if settings_file == args.network_settings_file and args.snapshot:
 227             continue
 228         if os.path.isfile(settings_file) is False:
 229             logging.error("Specified settings file does not "
 230                           "exist: {}".format(settings_file))
 231             raise ApexDeployException('Specified settings file does not '
 232                                       'exist: {}'.format(settings_file))
 233
 234
 235 def main():
 236     parser = create_deploy_parser()
 237     args = parser.parse_args(sys.argv[1:])
 238     # FIXME (trozet): this is only needed as a workaround for CI.  Remove
 239     # when CI is changed
 240     if os.getenv('IMAGES', False):
 241         args.image_dir = os.getenv('IMAGES')
 242     if args.debug:
 243         log_level = logging.DEBUG
 244     else:
 245         log_level = logging.INFO
 246     os.makedirs(os.path.dirname(args.log_file), exist_ok=True)
 247     formatter = '%(asctime)s %(levelname)s: %(message)s'
 248     logging.basicConfig(filename=args.log_file,
 249                         format=formatter,
 250                         datefmt='%m/%d/%Y %I:%M:%S %p',
 251                         level=log_level)
 252     console = logging.StreamHandler()
 253     console.setLevel(log_level)
 254     console.setFormatter(logging.Formatter(formatter))
 255     logging.getLogger('').addHandler(console)
 256     utils.install_ansible()
 257     validate_deploy_args(args)
 258     # Parse all settings
 259     deploy_settings = DeploySettings(args.deploy_settings_file)
 260     logging.info("Deploy settings are:\n {}".format(pprint.pformat(
 261         deploy_settings)))
 262
 263     if not args.snapshot:
 264         net_settings = NetworkSettings(args.network_settings_file)
 265         logging.info("Network settings are:\n {}".format(pprint.pformat(
 266             net_settings)))
 267         os_version = deploy_settings['deploy_options']['os_version']
 268         net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE)
 269         net_env = NetworkEnvironment(net_settings, net_env_file,
 270                                      os_version=os_version)
 271         net_env_target = os.path.join(APEX_TEMP_DIR, constants.NET_ENV_FILE)
 272         utils.dump_yaml(dict(net_env), net_env_target)
 273
 274         # get global deploy params
 275         ha_enabled = deploy_settings['global_params']['ha_enabled']
 276         introspect = deploy_settings['global_params'].get('introspect', True)
 277         net_list = net_settings.enabled_network_list
 278         if args.virtual:
 279             if args.virt_compute_ram is None:
 280                 compute_ram = args.virt_default_ram
 281             else:
 282                 compute_ram = args.virt_compute_ram
 283             if (deploy_settings['deploy_options']['sdn_controller'] ==
 284                     'opendaylight' and args.virt_default_ram < 12):
 285                 control_ram = 12
 286                 logging.warning('RAM per controller is too low.  OpenDaylight '
 287                                 'requires at least 12GB per controller.')
 288                 logging.info('Increasing RAM per controller to 12GB')
 289             elif args.virt_default_ram < 10:
 290                 control_ram = 10
 291                 logging.warning('RAM per controller is too low.  nosdn '
 292                                 'requires at least 10GB per controller.')
 293                 logging.info('Increasing RAM per controller to 10GB')
 294             else:
 295                 control_ram = args.virt_default_ram
 296             if ha_enabled and args.virt_compute_nodes < 2:
 297                 logging.debug(
 298                     'HA enabled, bumping number of compute nodes to 2')
 299                 args.virt_compute_nodes = 2
 300             virt_utils.generate_inventory(args.inventory_file, ha_enabled,
 301                                           num_computes=args.virt_compute_nodes,
 302                                           controller_ram=control_ram * 1024,
 303                                           compute_ram=compute_ram * 1024,
 304                                           vcpus=args.virt_cpus
 305                                           )
 306         inventory = Inventory(args.inventory_file, ha_enabled, args.virtual)
 307         logging.info("Inventory is:\n {}".format(pprint.pformat(
 308             inventory)))
 309
 310         validate_cross_settings(deploy_settings, net_settings, inventory)
 311     else:
 312         # only one network with snapshots
 313         net_list = [constants.ADMIN_NETWORK]
 314
 315     ds_opts = deploy_settings['deploy_options']
 316     ansible_args = {
 317         'virsh_enabled_networks': net_list,
 318         'snapshot': args.snapshot
 319     }
 320     utils.run_ansible(ansible_args,
 321                       os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 322                                    'deploy_dependencies.yml'))
 323     if args.snapshot:
 324         # Start snapshot Deployment
 325         logging.info('Executing Snapshot Deployment...')
 326         SnapshotDeployment(deploy_settings=deploy_settings,
 327                            snap_cache_dir=args.snap_cache,
 328                            fetch=not args.no_fetch,
 329                            all_in_one=not bool(args.virt_compute_nodes))
 330     else:
 331         # Start Standard TripleO Deployment
 332         deployment = ApexDeployment(deploy_settings, args.patches_file,
 333                                     args.deploy_settings_file)
 334         # TODO (trozet): add logic back from:
 335         # Iedb75994d35b5dc1dd5d5ce1a57277c8f3729dfd (FDIO DVR)
 336         uc_external = False
 337         if 'external' in net_settings.enabled_network_list:
 338             uc_external = True
 339         if args.virtual:
 340             # create all overcloud VMs
 341             build_vms(inventory, net_settings, args.deploy_dir)
 342         else:
 343             # Attach interfaces to jumphost for baremetal deployment
 344             jump_networks = ['admin']
 345             if uc_external:
 346                 jump_networks.append('external')
 347             for network in jump_networks:
 348                 if network == 'external':
 349                     # TODO(trozet): enable vlan secondary external networks
 350                     iface = net_settings['networks'][network][0][
 351                         'installer_vm']['members'][0]
 352                 else:
 353                     iface = net_settings['networks'][network]['installer_vm'][
 354                         'members'][0]
 355                 bridge = "br-{}".format(network)
 356                 jumphost.attach_interface_to_ovs(bridge, iface, network)
 357         instackenv_json = os.path.join(APEX_TEMP_DIR, 'instackenv.json')
 358         with open(instackenv_json, 'w') as fh:
 359             json.dump(inventory, fh)
 360
 361         # Create and configure undercloud
 362         if args.debug:
 363             root_pw = constants.DEBUG_OVERCLOUD_PW
 364         else:
 365             root_pw = None
 366
 367         if not args.upstream:
 368             logging.warning("Using upstream is now required for Apex. "
 369                             "Forcing upstream to true")
 370         if os_version == 'master':
 371             branch = 'master'
 372         else:
 373             branch = "stable/{}".format(os_version)
 374
 375         logging.info("Deploying with upstream artifacts for OpenStack "
 376                      "{}".format(os_version))
 377         args.image_dir = os.path.join(args.image_dir, os_version)
 378         upstream_url = constants.UPSTREAM_RDO.replace(
 379             constants.DEFAULT_OS_VERSION, os_version)
 380         upstream_targets = ['overcloud-full.tar', 'undercloud.qcow2']
 381         utils.fetch_upstream_and_unpack(args.image_dir, upstream_url,
 382                                         upstream_targets,
 383                                         fetch=not args.no_fetch)
 384         sdn_image = os.path.join(args.image_dir, 'overcloud-full.qcow2')
 385         # copy undercloud so we don't taint upstream fetch
 386         uc_image = os.path.join(args.image_dir, 'undercloud_mod.qcow2')
 387         uc_fetch_img = os.path.join(args.image_dir, 'undercloud.qcow2')
 388         shutil.copyfile(uc_fetch_img, uc_image)
 389         # prep undercloud with required packages
 390         uc_builder.add_upstream_packages(uc_image)
 391         uc_builder.inject_calipso_installer(APEX_TEMP_DIR, uc_image)
 392         # add patches from upstream to undercloud and overcloud
 393         logging.info('Adding patches to undercloud')
 394         patches = deployment.determine_patches()
 395         c_builder.add_upstream_patches(patches['undercloud'], uc_image,
 396                                        APEX_TEMP_DIR, branch)
 397
 398         # Create/Start Undercloud VM
 399         undercloud = uc_lib.Undercloud(args.image_dir,
 400                                        args.deploy_dir,
 401                                        root_pw=root_pw,
 402                                        external_network=uc_external,
 403                                        image_name=os.path.basename(uc_image),
 404                                        os_version=os_version)
 405         undercloud.start()
 406         undercloud_admin_ip = net_settings['networks'][
 407             constants.ADMIN_NETWORK]['installer_vm']['ip']
 408
 409         if ds_opts['containers']:
 410             tag = constants.DOCKER_TAG
 411         else:
 412             tag = None
 413
 414         # Generate nic templates
 415         for role in 'compute', 'controller':
 416             oc_cfg.create_nic_template(net_settings, deploy_settings, role,
 417                                        args.deploy_dir, APEX_TEMP_DIR)
 418         # Install Undercloud
 419         undercloud.configure(net_settings, deploy_settings,
 420                              os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 421                                           'configure_undercloud.yml'),
 422                              APEX_TEMP_DIR, virtual_oc=args.virtual)
 423
 424         # Prepare overcloud-full.qcow2
 425         logging.info("Preparing Overcloud for deployment...")
 426         if os_version != 'ocata':
 427             net_data_file = os.path.join(APEX_TEMP_DIR, 'network_data.yaml')
 428             net_data = network_data.create_network_data(net_settings,
 429                                                         net_data_file)
 430         else:
 431             net_data = False
 432
 433         shutil.copyfile(os.path.join(args.deploy_dir, 'build_ovs_nsh.sh'),
 434                         os.path.join(APEX_TEMP_DIR, 'build_ovs_nsh.sh'))
 435
 436         # TODO(trozet): Either fix opnfv env or default to use upstream env
 437         if args.env_file == 'opnfv-environment.yaml':
 438             # Override the env_file if it is defaulted to opnfv
 439             # opnfv env file will not work with upstream
 440             args.env_file = 'upstream-environment.yaml'
 441         opnfv_env = os.path.join(args.deploy_dir, args.env_file)
 442         oc_deploy.prep_env(deploy_settings, net_settings, inventory,
 443                            opnfv_env, net_env_target, APEX_TEMP_DIR)
 444         if not args.virtual:
 445             oc_deploy.LOOP_DEVICE_SIZE = "50G"
 446         patched_containers = oc_deploy.prep_image(
 447             deploy_settings, net_settings, sdn_image, APEX_TEMP_DIR,
 448             root_pw=root_pw, docker_tag=tag, patches=patches['overcloud'])
 449
 450         oc_deploy.create_deploy_cmd(deploy_settings, net_settings, inventory,
 451                                     APEX_TEMP_DIR, args.virtual,
 452                                     os.path.basename(opnfv_env),
 453                                     net_data=net_data)
 454         # Prepare undercloud with containers
 455         docker_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 456                                        'prepare_overcloud_containers.yml')
 457         if ds_opts['containers']:
 458             logging.info("Preparing Undercloud with Docker containers")
 459             sdn_env = oc_deploy.get_docker_sdn_files(ds_opts)
 460             sdn_env_files = str()
 461             for sdn_file in sdn_env:
 462                 sdn_env_files += " -e {}".format(sdn_file)
 463             if patched_containers:
 464                 oc_builder.archive_docker_patches(APEX_TEMP_DIR)
 465             container_vars = dict()
 466             container_vars['apex_temp_dir'] = APEX_TEMP_DIR
 467             container_vars['patched_docker_services'] = list(
 468                 patched_containers)
 469             container_vars['container_tag'] = constants.DOCKER_TAG
 470             container_vars['stackrc'] = 'source /home/stack/stackrc'
 471             container_vars['sdn'] = ds_opts['sdn_controller']
 472             container_vars['undercloud_ip'] = undercloud_admin_ip
 473             container_vars['os_version'] = os_version
 474             container_vars['aarch64'] = platform.machine() == 'aarch64'
 475             container_vars['sdn_env_file'] = sdn_env_files
 476             try:
 477                 utils.run_ansible(container_vars, docker_playbook,
 478                                   host=undercloud.ip, user='stack',
 479                                   tmp_dir=APEX_TEMP_DIR)
 480                 logging.info("Container preparation complete")
 481             except Exception:
 482                 logging.error("Unable to complete container prep on "
 483                               "Undercloud")
 484                 os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
 485                 raise
 486
 487         deploy_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 488                                        'deploy_overcloud.yml')
 489         virt_env = 'virtual-environment.yaml'
 490         bm_env = 'baremetal-environment.yaml'
 491         k8s_env = 'kubernetes-environment.yaml'
 492         for p_env in virt_env, bm_env, k8s_env:
 493             shutil.copyfile(os.path.join(args.deploy_dir, p_env),
 494                             os.path.join(APEX_TEMP_DIR, p_env))
 495
 496         # Start Overcloud Deployment
 497         logging.info("Executing Overcloud Deployment...")
 498         deploy_vars = dict()
 499         deploy_vars['virtual'] = args.virtual
 500         deploy_vars['debug'] = args.debug
 501         deploy_vars['aarch64'] = platform.machine() == 'aarch64'
 502         deploy_vars['introspect'] = not (args.virtual or
 503                                          deploy_vars['aarch64'] or
 504                                          not introspect)
 505         deploy_vars['dns_server_args'] = ''
 506         deploy_vars['apex_temp_dir'] = APEX_TEMP_DIR
 507         deploy_vars['apex_env_file'] = os.path.basename(opnfv_env)
 508         deploy_vars['stackrc'] = 'source /home/stack/stackrc'
 509         deploy_vars['overcloudrc'] = 'source /home/stack/overcloudrc'
 510         deploy_vars['undercloud_ip'] = undercloud_admin_ip
 511         deploy_vars['ha_enabled'] = ha_enabled
 512         deploy_vars['os_version'] = os_version
 513         deploy_vars['http_proxy'] = net_settings.get('http_proxy', '')
 514         deploy_vars['https_proxy'] = net_settings.get('https_proxy', '')
 515         deploy_vars['vim'] = ds_opts['vim']
 516         for dns_server in net_settings['dns_servers']:
 517             deploy_vars['dns_server_args'] += " --dns-nameserver {}".format(
 518                 dns_server)
 519         try:
 520             utils.run_ansible(deploy_vars, deploy_playbook, host=undercloud.ip,
 521                               user='stack', tmp_dir=APEX_TEMP_DIR)
 522             logging.info("Overcloud deployment complete")
 523         except Exception:
 524             logging.error("Deployment Failed.  Please check deploy log as "
 525                           "well as mistral logs in "
 526                           "{}".format(os.path.join(APEX_TEMP_DIR,
 527                                                    'mistral_logs.tar.gz')))
 528             raise
 529         finally:
 530             os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
 531
 532         # Post install
 533         logging.info("Executing post deploy configuration")
 534         jumphost.configure_bridges(net_settings)
 535         nova_output = os.path.join(APEX_TEMP_DIR, 'nova_output')
 536         deploy_vars['overcloud_nodes'] = parsers.parse_nova_output(
 537             nova_output)
 538         deploy_vars['SSH_OPTIONS'] = '-o StrictHostKeyChecking=no -o ' \
 539                                      'GlobalKnownHostsFile=/dev/null -o ' \
 540                                      'UserKnownHostsFile=/dev/null -o ' \
 541                                      'LogLevel=error'
 542         deploy_vars['external_network_cmds'] = \
 543             oc_deploy.external_network_cmds(net_settings, deploy_settings)
 544         # TODO(trozet): just parse all ds_opts as deploy vars one time
 545         deploy_vars['gluon'] = ds_opts['gluon']
 546         deploy_vars['sdn'] = ds_opts['sdn_controller']
 547         for dep_option in 'yardstick', 'dovetail', 'vsperf':
 548             if dep_option in ds_opts:
 549                 deploy_vars[dep_option] = ds_opts[dep_option]
 550             else:
 551                 deploy_vars[dep_option] = False
 552         deploy_vars['dataplane'] = ds_opts['dataplane']
 553         overcloudrc = os.path.join(APEX_TEMP_DIR, 'overcloudrc')
 554         if ds_opts['congress']:
 555             deploy_vars['congress_datasources'] = \
 556                 oc_deploy.create_congress_cmds(overcloudrc)
 557             deploy_vars['congress'] = True
 558         else:
 559             deploy_vars['congress'] = False
 560         deploy_vars['calipso'] = ds_opts.get('calipso', False)
 561         deploy_vars['calipso_ip'] = undercloud_admin_ip
 562         # overcloudrc.v3 removed and set as default in queens and later
 563         if os_version == 'pike':
 564             deploy_vars['overcloudrc_files'] = ['overcloudrc',
 565                                                 'overcloudrc.v3']
 566         else:
 567             deploy_vars['overcloudrc_files'] = ['overcloudrc']
 568
 569         post_undercloud = os.path.join(args.lib_dir,
 570                                        constants.ANSIBLE_PATH,
 571                                        'post_deploy_undercloud.yml')
 572         logging.info("Executing post deploy configuration undercloud "
 573                      "playbook")
 574         try:
 575             utils.run_ansible(deploy_vars, post_undercloud,
 576                               host=undercloud.ip, user='stack',
 577                               tmp_dir=APEX_TEMP_DIR)
 578             logging.info("Post Deploy Undercloud Configuration Complete")
 579         except Exception:
 580             logging.error("Post Deploy Undercloud Configuration failed.  "
 581                           "Please check log")
 582             raise
 583
 584         # Deploy kubernetes if enabled
 585         # (TODO)zshi move handling of kubernetes deployment
 586         # to its own deployment class
 587         if deploy_vars['vim'] == 'k8s':
 588             # clone kubespray repo
 589             git.Repo.clone_from(constants.KUBESPRAY_URL,
 590                                 os.path.join(APEX_TEMP_DIR, 'kubespray'))
 591             shutil.copytree(
 592                 os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
 593                              'sample'),
 594                 os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
 595                              'apex'))
 596             k8s_node_inventory = {
 597                 'all':
 598                     {'hosts': {},
 599                      'children': {
 600                          'k8s-cluster': {
 601                              'children': {
 602                                  'kube-master': {
 603                                      'hosts': {}
 604                                  },
 605                                  'kube-node': {
 606                                      'hosts': {}
 607                                  }
 608                              }
 609                          },
 610                          'etcd': {
 611                              'hosts': {}
 612                          }
 613                     }
 614                     }
 615             }
 616             for node, ip in deploy_vars['overcloud_nodes'].items():
 617                 k8s_node_inventory['all']['hosts'][node] = {
 618                     'ansible_become': True,
 619                     'ansible_ssh_host': ip,
 620                     'ansible_become_user': 'root',
 621                     'ip': ip
 622                 }
 623                 if 'controller' in node:
 624                     k8s_node_inventory['all']['children']['k8s-cluster'][
 625                         'children']['kube-master']['hosts'][node] = None
 626                     k8s_node_inventory['all']['children']['etcd'][
 627                         'hosts'][node] = None
 628                 elif 'compute' in node:
 629                     k8s_node_inventory['all']['children']['k8s-cluster'][
 630                         'children']['kube-node']['hosts'][node] = None
 631
 632             kubespray_dir = os.path.join(APEX_TEMP_DIR, 'kubespray')
 633             with open(os.path.join(kubespray_dir, 'inventory', 'apex',
 634                                    'apex.yaml'), 'w') as invfile:
 635                 yaml.dump(k8s_node_inventory, invfile,
 636                           default_flow_style=False)
 637             k8s_deploy_vars = {}
 638             # Add kubespray ansible control variables in k8s_deploy_vars,
 639             # example: 'kube_network_plugin': 'flannel'
 640             k8s_deploy = os.path.join(kubespray_dir, 'cluster.yml')
 641             k8s_deploy_inv_file = os.path.join(kubespray_dir, 'inventory',
 642                                                'apex', 'apex.yaml')
 643
 644             k8s_remove_pkgs = os.path.join(args.lib_dir,
 645                                            constants.ANSIBLE_PATH,
 646                                            'k8s_remove_pkgs.yml')
 647             try:
 648                 logging.debug("Removing any existing overcloud docker "
 649                               "packages")
 650                 utils.run_ansible(k8s_deploy_vars, k8s_remove_pkgs,
 651                                   host=k8s_deploy_inv_file,
 652                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 653                 logging.info("k8s Deploy Remove Existing Docker Related "
 654                              "Packages Complete")
 655             except Exception:
 656                 logging.error("k8s Deploy Remove Existing Docker Related "
 657                               "Packages failed. Please check log")
 658                 raise
 659
 660             try:
 661                 utils.run_ansible(k8s_deploy_vars, k8s_deploy,
 662                                   host=k8s_deploy_inv_file,
 663                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 664                 logging.info("k8s Deploy Overcloud Configuration Complete")
 665             except Exception:
 666                 logging.error("k8s Deploy Overcloud Configuration failed."
 667                               "Please check log")
 668                 raise
 669
 670         # Post deploy overcloud node configuration
 671         # TODO(trozet): just parse all ds_opts as deploy vars one time
 672         deploy_vars['sfc'] = ds_opts['sfc']
 673         deploy_vars['vpn'] = ds_opts['vpn']
 674         deploy_vars['l2gw'] = ds_opts.get('l2gw')
 675         deploy_vars['sriov'] = ds_opts.get('sriov')
 676         deploy_vars['tacker'] = ds_opts.get('tacker')
 677         # TODO(trozet): pull all logs and store in tmp dir in overcloud
 678         # playbook
 679         post_overcloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
 680                                       'post_deploy_overcloud.yml')
 681         # Run per overcloud node
 682         for node, ip in deploy_vars['overcloud_nodes'].items():
 683             logging.info("Executing Post deploy overcloud playbook on "
 684                          "node {}".format(node))
 685             try:
 686                 utils.run_ansible(deploy_vars, post_overcloud, host=ip,
 687                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 688                 logging.info("Post Deploy Overcloud Configuration Complete "
 689                              "for node {}".format(node))
 690             except Exception:
 691                 logging.error("Post Deploy Overcloud Configuration failed "
 692                               "for node {}. Please check log".format(node))
 693                 raise
 694         logging.info("Apex deployment complete")
 695         logging.info("Undercloud IP: {}, please connect by doing "
 696                      "'opnfv-util undercloud'".format(undercloud.ip))
 697         # TODO(trozet): add logging here showing controller VIP and horizon url
 698
 699
 700 if __name__ == '__main__':
 701     main()