apex/deploy.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 import argparse
  13 import json
  14 import logging
  15 import os
  16 import platform
  17 import pprint
  18 import shutil
  19 import sys
  20 import tempfile
  21
  22 import apex.virtual.configure_vm as vm_lib
  23 import apex.virtual.utils as virt_utils
  24 from apex import DeploySettings
  25 from apex import Inventory
  26 from apex import NetworkEnvironment
  27 from apex import NetworkSettings
  28 from apex.builders import common_builder as c_builder
  29 from apex.builders import overcloud_builder as oc_builder
  30 from apex.builders import undercloud_builder as uc_builder
  31 from apex.common import utils
  32 from apex.common import constants
  33 from apex.common import parsers
  34 from apex.common.exceptions import ApexDeployException
  35 from apex.network import jumphost
  36 from apex.network import network_data
  37 from apex.undercloud import undercloud as uc_lib
  38 from apex.overcloud import config as oc_cfg
  39 from apex.overcloud import deploy as oc_deploy
  40
  41 APEX_TEMP_DIR = tempfile.mkdtemp(prefix='apex_tmp')
  42 ANSIBLE_PATH = 'ansible/playbooks'
  43 SDN_IMAGE = 'overcloud-full-opendaylight.qcow2'
  44
  45
  46 def deploy_quickstart(args, deploy_settings_file, network_settings_file,
  47                       inventory_file=None):
  48     pass
  49
  50
  51 def validate_cross_settings(deploy_settings, net_settings, inventory):
  52     """
  53     Used to validate compatibility across settings file.
  54     :param deploy_settings: parsed settings for deployment
  55     :param net_settings: parsed settings for network
  56     :param inventory: parsed inventory file
  57     :return: None
  58     """
  59
  60     if deploy_settings['deploy_options']['dataplane'] != 'ovs' and 'tenant' \
  61             not in net_settings.enabled_network_list:
  62         raise ApexDeployException("Setting a DPDK based dataplane requires"
  63                                   "a dedicated NIC for tenant network")
  64
  65     if 'odl_vpp_routing_node' in deploy_settings['deploy_options']:
  66         if deploy_settings['deploy_options']['dataplane'] != 'fdio':
  67             raise ApexDeployException("odl_vpp_routing_node should only be set"
  68                                       "when dataplane is set to fdio")
  69         if deploy_settings['deploy_options'].get('dvr') is True:
  70             raise ApexDeployException("odl_vpp_routing_node should only be set"
  71                                       "when dvr is not enabled")
  72
  73     # TODO(trozet): add more checks here like RAM for ODL, etc
  74     # check if odl_vpp_netvirt is true and vpp is set
  75     # Check if fdio and nosdn:
  76     # tenant_nic_mapping_controller_members" ==
  77     # "$tenant_nic_mapping_compute_members
  78
  79
  80 def build_vms(inventory, network_settings,
  81               template_dir='/usr/share/opnfv-apex'):
  82     """
  83     Creates VMs and configures vbmc and host
  84     :param inventory:
  85     :param network_settings:
  86     :return:
  87     """
  88
  89     for idx, node in enumerate(inventory['nodes']):
  90         name = 'baremetal{}'.format(idx)
  91         volume = name + ".qcow2"
  92         volume_path = os.path.join(constants.LIBVIRT_VOLUME_PATH, volume)
  93         # TODO(trozet): add error checking
  94         vm_lib.create_vm(
  95             name, volume_path,
  96             baremetal_interfaces=network_settings.enabled_network_list,
  97             memory=node['memory'], cpus=node['cpu'],
  98             macs=node['mac'],
  99             template_dir=template_dir)
 100         virt_utils.host_setup({name: node['pm_port']})
 101
 102
 103 def create_deploy_parser():
 104     deploy_parser = argparse.ArgumentParser()
 105     deploy_parser.add_argument('--debug', action='store_true', default=False,
 106                                help="Turn on debug messages")
 107     deploy_parser.add_argument('-l', '--log-file',
 108                                default='./apex_deploy.log',
 109                                dest='log_file', help="Log file to log to")
 110     deploy_parser.add_argument('-d', '--deploy-settings',
 111                                dest='deploy_settings_file',
 112                                required=True,
 113                                help='File which contains Apex deploy settings')
 114     deploy_parser.add_argument('-n', '--network-settings',
 115                                dest='network_settings_file',
 116                                required=True,
 117                                help='File which contains Apex network '
 118                                     'settings')
 119     deploy_parser.add_argument('-i', '--inventory-file',
 120                                dest='inventory_file',
 121                                default=None,
 122                                help='Inventory file which contains POD '
 123                                     'definition')
 124     deploy_parser.add_argument('-e', '--environment-file',
 125                                dest='env_file',
 126                                default='opnfv-environment.yaml',
 127                                help='Provide alternate base env file located '
 128                                     'in deploy_dir')
 129     deploy_parser.add_argument('-v', '--virtual', action='store_true',
 130                                default=False,
 131                                dest='virtual',
 132                                help='Enable virtual deployment')
 133     deploy_parser.add_argument('--interactive', action='store_true',
 134                                default=False,
 135                                help='Enable interactive deployment mode which '
 136                                     'requires user to confirm steps of '
 137                                     'deployment')
 138     deploy_parser.add_argument('--virtual-computes',
 139                                dest='virt_compute_nodes',
 140                                default=1,
 141                                type=int,
 142                                help='Number of Virtual Compute nodes to create'
 143                                     ' and use during deployment (defaults to 1'
 144                                     ' for noha and 2 for ha)')
 145     deploy_parser.add_argument('--virtual-cpus',
 146                                dest='virt_cpus',
 147                                default=4,
 148                                type=int,
 149                                help='Number of CPUs to use per Overcloud VM in'
 150                                     ' a virtual deployment (defaults to 4)')
 151     deploy_parser.add_argument('--virtual-default-ram',
 152                                dest='virt_default_ram',
 153                                default=8,
 154                                type=int,
 155                                help='Amount of default RAM to use per '
 156                                     'Overcloud VM in GB (defaults to 8).')
 157     deploy_parser.add_argument('--virtual-compute-ram',
 158                                dest='virt_compute_ram',
 159                                default=None,
 160                                type=int,
 161                                help='Amount of RAM to use per Overcloud '
 162                                     'Compute VM in GB (defaults to 8). '
 163                                     'Overrides --virtual-default-ram arg for '
 164                                     'computes')
 165     deploy_parser.add_argument('--deploy-dir',
 166                                default='/usr/share/opnfv-apex',
 167                                help='Directory to deploy from which contains '
 168                                     'base config files for deployment')
 169     deploy_parser.add_argument('--image-dir',
 170                                default='/var/opt/opnfv/images',
 171                                help='Directory which contains '
 172                                     'base disk images for deployment')
 173     deploy_parser.add_argument('--lib-dir',
 174                                default='/usr/share/opnfv-apex',
 175                                help='Directory path for apex ansible '
 176                                     'and third party libs')
 177     deploy_parser.add_argument('--quickstart', action='store_true',
 178                                default=False,
 179                                help='Use tripleo-quickstart to deploy')
 180     deploy_parser.add_argument('--upstream', action='store_true',
 181                                default=False,
 182                                help='Force deployment to use upstream '
 183                                     'artifacts')
 184     return deploy_parser
 185
 186
 187 def validate_deploy_args(args):
 188     """
 189     Validates arguments for deploy
 190     :param args:
 191     :return: None
 192     """
 193
 194     logging.debug('Validating arguments for deployment')
 195     if args.virtual and args.inventory_file is not None:
 196         logging.error("Virtual enabled but inventory file also given")
 197         raise ApexDeployException('You should not specify an inventory file '
 198                                   'with virtual deployments')
 199     elif args.virtual:
 200         args.inventory_file = os.path.join(APEX_TEMP_DIR,
 201                                            'inventory-virt.yaml')
 202     elif os.path.isfile(args.inventory_file) is False:
 203         logging.error("Specified inventory file does not exist: {}".format(
 204             args.inventory_file))
 205         raise ApexDeployException('Specified inventory file does not exist')
 206
 207     for settings_file in (args.deploy_settings_file,
 208                           args.network_settings_file):
 209         if os.path.isfile(settings_file) is False:
 210             logging.error("Specified settings file does not "
 211                           "exist: {}".format(settings_file))
 212             raise ApexDeployException('Specified settings file does not '
 213                                       'exist: {}'.format(settings_file))
 214
 215
 216 def main():
 217     parser = create_deploy_parser()
 218     args = parser.parse_args(sys.argv[1:])
 219     # FIXME (trozet): this is only needed as a workaround for CI.  Remove
 220     # when CI is changed
 221     if os.getenv('IMAGES', False):
 222         args.image_dir = os.getenv('IMAGES')
 223     if args.debug:
 224         log_level = logging.DEBUG
 225     else:
 226         log_level = logging.INFO
 227     os.makedirs(os.path.dirname(args.log_file), exist_ok=True)
 228     formatter = '%(asctime)s %(levelname)s: %(message)s'
 229     logging.basicConfig(filename=args.log_file,
 230                         format=formatter,
 231                         datefmt='%m/%d/%Y %I:%M:%S %p',
 232                         level=log_level)
 233     console = logging.StreamHandler()
 234     console.setLevel(log_level)
 235     console.setFormatter(logging.Formatter(formatter))
 236     logging.getLogger('').addHandler(console)
 237     utils.install_ansible()
 238     validate_deploy_args(args)
 239     # Parse all settings
 240     deploy_settings = DeploySettings(args.deploy_settings_file)
 241     logging.info("Deploy settings are:\n {}".format(pprint.pformat(
 242                  deploy_settings)))
 243     net_settings = NetworkSettings(args.network_settings_file)
 244     logging.info("Network settings are:\n {}".format(pprint.pformat(
 245                  net_settings)))
 246     os_version = deploy_settings['deploy_options']['os_version']
 247     net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE)
 248     net_env = NetworkEnvironment(net_settings, net_env_file,
 249                                  os_version=os_version)
 250     net_env_target = os.path.join(APEX_TEMP_DIR, constants.NET_ENV_FILE)
 251     utils.dump_yaml(dict(net_env), net_env_target)
 252
 253     # get global deploy params
 254     ha_enabled = deploy_settings['global_params']['ha_enabled']
 255     introspect = deploy_settings['global_params'].get('introspect', True)
 256
 257     if args.virtual:
 258         if args.virt_compute_ram is None:
 259             compute_ram = args.virt_default_ram
 260         else:
 261             compute_ram = args.virt_compute_ram
 262         if deploy_settings['deploy_options']['sdn_controller'] == \
 263                 'opendaylight' and args.virt_default_ram < 12:
 264             control_ram = 12
 265             logging.warning('RAM per controller is too low.  OpenDaylight '
 266                             'requires at least 12GB per controller.')
 267             logging.info('Increasing RAM per controller to 12GB')
 268         elif args.virt_default_ram < 10:
 269             control_ram = 10
 270             logging.warning('RAM per controller is too low.  nosdn '
 271                             'requires at least 10GB per controller.')
 272             logging.info('Increasing RAM per controller to 10GB')
 273         else:
 274             control_ram = args.virt_default_ram
 275         if ha_enabled and args.virt_compute_nodes < 2:
 276             logging.debug('HA enabled, bumping number of compute nodes to 2')
 277             args.virt_compute_nodes = 2
 278         virt_utils.generate_inventory(args.inventory_file, ha_enabled,
 279                                       num_computes=args.virt_compute_nodes,
 280                                       controller_ram=control_ram * 1024,
 281                                       compute_ram=compute_ram * 1024,
 282                                       vcpus=args.virt_cpus
 283                                       )
 284     inventory = Inventory(args.inventory_file, ha_enabled, args.virtual)
 285
 286     validate_cross_settings(deploy_settings, net_settings, inventory)
 287     ds_opts = deploy_settings['deploy_options']
 288     if args.quickstart:
 289         deploy_settings_file = os.path.join(APEX_TEMP_DIR,
 290                                             'apex_deploy_settings.yaml')
 291         utils.dump_yaml(utils.dict_objects_to_str(deploy_settings),
 292                         deploy_settings_file)
 293         logging.info("File created: {}".format(deploy_settings_file))
 294         network_settings_file = os.path.join(APEX_TEMP_DIR,
 295                                              'apex_network_settings.yaml')
 296         utils.dump_yaml(utils.dict_objects_to_str(net_settings),
 297                         network_settings_file)
 298         logging.info("File created: {}".format(network_settings_file))
 299         deploy_quickstart(args, deploy_settings_file, network_settings_file,
 300                           args.inventory_file)
 301     else:
 302         # TODO (trozet): add logic back from:
 303         # Iedb75994d35b5dc1dd5d5ce1a57277c8f3729dfd (FDIO DVR)
 304         ansible_args = {
 305             'virsh_enabled_networks': net_settings.enabled_network_list
 306         }
 307         utils.run_ansible(ansible_args,
 308                           os.path.join(args.lib_dir, ANSIBLE_PATH,
 309                                        'deploy_dependencies.yml'))
 310         uc_external = False
 311         if 'external' in net_settings.enabled_network_list:
 312             uc_external = True
 313         if args.virtual:
 314             # create all overcloud VMs
 315             build_vms(inventory, net_settings, args.deploy_dir)
 316         else:
 317             # Attach interfaces to jumphost for baremetal deployment
 318             jump_networks = ['admin']
 319             if uc_external:
 320                 jump_networks.append('external')
 321             for network in jump_networks:
 322                 if network == 'external':
 323                     # TODO(trozet): enable vlan secondary external networks
 324                     iface = net_settings['networks'][network][0][
 325                         'installer_vm']['members'][0]
 326                 else:
 327                     iface = net_settings['networks'][network]['installer_vm'][
 328                         'members'][0]
 329                 bridge = "br-{}".format(network)
 330                 jumphost.attach_interface_to_ovs(bridge, iface, network)
 331         instackenv_json = os.path.join(APEX_TEMP_DIR, 'instackenv.json')
 332         with open(instackenv_json, 'w') as fh:
 333             json.dump(inventory, fh)
 334
 335         # Create and configure undercloud
 336         if args.debug:
 337             root_pw = constants.DEBUG_OVERCLOUD_PW
 338         else:
 339             root_pw = None
 340
 341         upstream = (os_version != constants.DEFAULT_OS_VERSION or
 342                     args.upstream)
 343         if os_version == 'master':
 344             branch = 'master'
 345         else:
 346             branch = "stable/{}".format(os_version)
 347         if upstream:
 348             logging.info("Deploying with upstream artifacts for OpenStack "
 349                          "{}".format(os_version))
 350             args.image_dir = os.path.join(args.image_dir, os_version)
 351             upstream_url = constants.UPSTREAM_RDO.replace(
 352                 constants.DEFAULT_OS_VERSION, os_version)
 353             upstream_targets = ['overcloud-full.tar', 'undercloud.qcow2']
 354             utils.fetch_upstream_and_unpack(args.image_dir, upstream_url,
 355                                             upstream_targets)
 356             sdn_image = os.path.join(args.image_dir, 'overcloud-full.qcow2')
 357             if ds_opts['sdn_controller'] == 'opendaylight':
 358                 logging.info("Preparing upstream image with OpenDaylight")
 359                 oc_builder.inject_opendaylight(
 360                     odl_version=ds_opts['odl_version'],
 361                     image=sdn_image,
 362                     tmp_dir=APEX_TEMP_DIR
 363                 )
 364             # copy undercloud so we don't taint upstream fetch
 365             uc_image = os.path.join(args.image_dir, 'undercloud_mod.qcow2')
 366             uc_fetch_img = os.path.join(args.image_dir, 'undercloud.qcow2')
 367             shutil.copyfile(uc_fetch_img, uc_image)
 368             # prep undercloud with required packages
 369             uc_builder.add_upstream_packages(uc_image)
 370             # add patches from upstream to undercloud and overcloud
 371             logging.info('Adding patches to undercloud')
 372             patches = deploy_settings['global_params']['patches']
 373             c_builder.add_upstream_patches(patches['undercloud'], uc_image,
 374                                            APEX_TEMP_DIR, branch)
 375             logging.info('Adding patches to overcloud')
 376             c_builder.add_upstream_patches(patches['overcloud'], sdn_image,
 377                                            APEX_TEMP_DIR, branch)
 378         else:
 379             sdn_image = os.path.join(args.image_dir, SDN_IMAGE)
 380             uc_image = 'undercloud.qcow2'
 381         undercloud = uc_lib.Undercloud(args.image_dir,
 382                                        args.deploy_dir,
 383                                        root_pw=root_pw,
 384                                        external_network=uc_external,
 385                                        image_name=os.path.basename(uc_image),
 386                                        os_version=os_version)
 387         undercloud.start()
 388
 389         # Generate nic templates
 390         for role in 'compute', 'controller':
 391             oc_cfg.create_nic_template(net_settings, deploy_settings, role,
 392                                        args.deploy_dir, APEX_TEMP_DIR)
 393         # Install Undercloud
 394         undercloud.configure(net_settings, deploy_settings,
 395                              os.path.join(args.lib_dir, ANSIBLE_PATH,
 396                                           'configure_undercloud.yml'),
 397                              APEX_TEMP_DIR)
 398
 399         # Prepare overcloud-full.qcow2
 400         logging.info("Preparing Overcloud for deployment...")
 401         if os_version != 'ocata':
 402             net_data_file = os.path.join(APEX_TEMP_DIR, 'network_data.yaml')
 403             net_data = network_data.create_network_data(net_settings,
 404                                                         net_data_file)
 405         else:
 406             net_data = False
 407         if upstream and args.env_file == 'opnfv-environment.yaml':
 408             # Override the env_file if it is defaulted to opnfv
 409             # opnfv env file will not work with upstream
 410             args.env_file = 'upstream-environment.yaml'
 411         opnfv_env = os.path.join(args.deploy_dir, args.env_file)
 412         if not upstream:
 413             oc_deploy.prep_env(deploy_settings, net_settings, inventory,
 414                                opnfv_env, net_env_target, APEX_TEMP_DIR)
 415             oc_deploy.prep_image(deploy_settings, net_settings, sdn_image,
 416                                  APEX_TEMP_DIR, root_pw=root_pw)
 417         else:
 418             shutil.copyfile(sdn_image, os.path.join(APEX_TEMP_DIR,
 419                                                     'overcloud-full.qcow2'))
 420             shutil.copyfile(
 421                 opnfv_env,
 422                 os.path.join(APEX_TEMP_DIR, os.path.basename(opnfv_env))
 423             )
 424
 425         oc_deploy.create_deploy_cmd(deploy_settings, net_settings, inventory,
 426                                     APEX_TEMP_DIR, args.virtual,
 427                                     os.path.basename(opnfv_env),
 428                                     net_data=net_data)
 429         deploy_playbook = os.path.join(args.lib_dir, ANSIBLE_PATH,
 430                                        'deploy_overcloud.yml')
 431         virt_env = 'virtual-environment.yaml'
 432         bm_env = 'baremetal-environment.yaml'
 433         for p_env in virt_env, bm_env:
 434             shutil.copyfile(os.path.join(args.deploy_dir, p_env),
 435                             os.path.join(APEX_TEMP_DIR, p_env))
 436
 437         # Start Overcloud Deployment
 438         logging.info("Executing Overcloud Deployment...")
 439         deploy_vars = dict()
 440         deploy_vars['virtual'] = args.virtual
 441         deploy_vars['debug'] = args.debug
 442         deploy_vars['aarch64'] = platform.machine() == 'aarch64'
 443         deploy_vars['introspect'] = not (args.virtual or
 444                                          deploy_vars['aarch64'] or
 445                                          not introspect)
 446         deploy_vars['dns_server_args'] = ''
 447         deploy_vars['apex_temp_dir'] = APEX_TEMP_DIR
 448         deploy_vars['apex_env_file'] = os.path.basename(opnfv_env)
 449         deploy_vars['stackrc'] = 'source /home/stack/stackrc'
 450         deploy_vars['overcloudrc'] = 'source /home/stack/overcloudrc'
 451         deploy_vars['upstream'] = upstream
 452         deploy_vars['os_version'] = os_version
 453         deploy_vars['http_proxy'] = net_settings.get('http_proxy', '')
 454         deploy_vars['https_proxy'] = net_settings.get('https_proxy', '')
 455         for dns_server in net_settings['dns_servers']:
 456             deploy_vars['dns_server_args'] += " --dns-nameserver {}".format(
 457                 dns_server)
 458         try:
 459             utils.run_ansible(deploy_vars, deploy_playbook, host=undercloud.ip,
 460                               user='stack', tmp_dir=APEX_TEMP_DIR)
 461             logging.info("Overcloud deployment complete")
 462         except Exception:
 463             logging.error("Deployment Failed.  Please check log")
 464             raise
 465         finally:
 466             os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
 467
 468         # Post install
 469         logging.info("Executing post deploy configuration")
 470         jumphost.configure_bridges(net_settings)
 471         nova_output = os.path.join(APEX_TEMP_DIR, 'nova_output')
 472         deploy_vars['overcloud_nodes'] = parsers.parse_nova_output(
 473             nova_output)
 474         deploy_vars['SSH_OPTIONS'] = '-o StrictHostKeyChecking=no -o ' \
 475                                      'GlobalKnownHostsFile=/dev/null -o ' \
 476                                      'UserKnownHostsFile=/dev/null -o ' \
 477                                      'LogLevel=error'
 478         deploy_vars['external_network_cmds'] = \
 479             oc_deploy.external_network_cmds(net_settings)
 480         # TODO(trozet): just parse all ds_opts as deploy vars one time
 481         deploy_vars['gluon'] = ds_opts['gluon']
 482         deploy_vars['sdn'] = ds_opts['sdn_controller']
 483         for dep_option in 'yardstick', 'dovetail', 'vsperf':
 484             if dep_option in ds_opts:
 485                 deploy_vars[dep_option] = ds_opts[dep_option]
 486             else:
 487                 deploy_vars[dep_option] = False
 488         deploy_vars['dataplane'] = ds_opts['dataplane']
 489         overcloudrc = os.path.join(APEX_TEMP_DIR, 'overcloudrc')
 490         if ds_opts['congress']:
 491             deploy_vars['congress_datasources'] = \
 492                 oc_deploy.create_congress_cmds(overcloudrc)
 493             deploy_vars['congress'] = True
 494         else:
 495             deploy_vars['congress'] = False
 496         deploy_vars['calipso'] = ds_opts.get('calipso', False)
 497         deploy_vars['calipso_ip'] = net_settings['networks']['admin'][
 498             'installer_vm']['ip']
 499         # TODO(trozet): this is probably redundant with getting external
 500         # network info from undercloud.py
 501         if 'external' in net_settings.enabled_network_list:
 502             ext_cidr = net_settings['networks']['external'][0]['cidr']
 503         else:
 504             ext_cidr = net_settings['networks']['admin']['cidr']
 505         deploy_vars['external_cidr'] = str(ext_cidr)
 506         if ext_cidr.version == 6:
 507             deploy_vars['external_network_ipv6'] = True
 508         else:
 509             deploy_vars['external_network_ipv6'] = False
 510         post_undercloud = os.path.join(args.lib_dir, ANSIBLE_PATH,
 511                                        'post_deploy_undercloud.yml')
 512         logging.info("Executing post deploy configuration undercloud playbook")
 513         try:
 514             utils.run_ansible(deploy_vars, post_undercloud, host=undercloud.ip,
 515                               user='stack', tmp_dir=APEX_TEMP_DIR)
 516             logging.info("Post Deploy Undercloud Configuration Complete")
 517         except Exception:
 518             logging.error("Post Deploy Undercloud Configuration failed.  "
 519                           "Please check log")
 520             raise
 521         # Post deploy overcloud node configuration
 522         # TODO(trozet): just parse all ds_opts as deploy vars one time
 523         deploy_vars['sfc'] = ds_opts['sfc']
 524         deploy_vars['vpn'] = ds_opts['vpn']
 525         deploy_vars['l2gw'] = ds_opts.get('l2gw')
 526         deploy_vars['sriov'] = ds_opts.get('sriov')
 527         # TODO(trozet): pull all logs and store in tmp dir in overcloud
 528         # playbook
 529         post_overcloud = os.path.join(args.lib_dir, ANSIBLE_PATH,
 530                                       'post_deploy_overcloud.yml')
 531         # Run per overcloud node
 532         for node, ip in deploy_vars['overcloud_nodes'].items():
 533             logging.info("Executing Post deploy overcloud playbook on "
 534                          "node {}".format(node))
 535             try:
 536                 utils.run_ansible(deploy_vars, post_overcloud, host=ip,
 537                                   user='heat-admin', tmp_dir=APEX_TEMP_DIR)
 538                 logging.info("Post Deploy Overcloud Configuration Complete "
 539                              "for node {}".format(node))
 540             except Exception:
 541                 logging.error("Post Deploy Overcloud Configuration failed "
 542                               "for node {}. Please check log".format(node))
 543                 raise
 544         logging.info("Apex deployment complete")
 545         logging.info("Undercloud IP: {}, please connect by doing "
 546                      "'opnfv-util undercloud'".format(undercloud.ip))
 547         # TODO(trozet): add logging here showing controller VIP and horizon url
 548
 549
 550 if __name__ == '__main__':
 551     main()