apex/overcloud/deploy.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 import base64
  11 import fileinput
  12 import logging
  13 import os
  14 import re
  15 import shutil
  16 import uuid
  17 import struct
  18 import time
  19
  20 from apex.common import constants as con
  21 from apex.common.exceptions import ApexDeployException
  22 from apex.common import parsers
  23 from apex.virtual import utils as virt_utils
  24 from cryptography.hazmat.primitives import serialization as \
  25     crypto_serialization
  26 from cryptography.hazmat.primitives.asymmetric import rsa
  27 from cryptography.hazmat.backends import default_backend as \
  28     crypto_default_backend
  29
  30
  31 SDN_FILE_MAP = {
  32     'opendaylight': {
  33         'sfc': 'neutron-sfc-opendaylight.yaml',
  34         'vpn': 'neutron-bgpvpn-opendaylight.yaml',
  35         'gluon': 'gluon.yaml',
  36         'vpp': {
  37             'odl_vpp_netvirt': 'neutron-opendaylight-netvirt-vpp.yaml',
  38             'dvr': 'neutron-opendaylight-fdio-dvr.yaml',
  39             'default': 'neutron-opendaylight-honeycomb.yaml'
  40         },
  41         'default': 'neutron-opendaylight.yaml',
  42     },
  43     'onos': {
  44         'sfc': 'neutron-onos-sfc.yaml',
  45         'default': 'neutron-onos.yaml'
  46     },
  47     'ovn': 'neutron-ml2-ovn.yaml',
  48     False: {
  49         'vpp': 'neutron-ml2-vpp.yaml',
  50         'dataplane': ('ovs_dpdk', 'neutron-ovs-dpdk.yaml')
  51     }
  52 }
  53
  54 OTHER_FILE_MAP = {
  55     'tacker': 'enable_tacker.yaml',
  56     'congress': 'enable_congress.yaml',
  57     'barometer': 'enable_barometer.yaml',
  58     'rt_kvm': 'enable_rt_kvm.yaml'
  59 }
  60
  61 OVS_PERF_MAP = {
  62     'HostCpusList': 'dpdk_cores',
  63     'NeutronDpdkCoreList': 'pmd_cores',
  64     'NeutronDpdkSocketMemory': 'socket_memory',
  65     'NeutronDpdkMemoryChannels': 'memory_channels'
  66 }
  67
  68 OVS_NSH_KMOD_RPM = "openvswitch-kmod-2.6.1-1.el7.centos.x86_64.rpm"
  69 OVS_NSH_RPM = "openvswitch-2.6.1-1.el7.centos.x86_64.rpm"
  70 ODL_NETVIRT_VPP_RPM = "/root/opendaylight-7.0.0-0.1.20170531snap665.el7" \
  71                       ".noarch.rpm"
  72
  73
  74 def build_sdn_env_list(ds, sdn_map, env_list=None):
  75     if env_list is None:
  76         env_list = list()
  77     for k, v in sdn_map.items():
  78         if ds['sdn_controller'] == k or (k in ds and ds[k] is True):
  79             if isinstance(v, dict):
  80                 env_list.extend(build_sdn_env_list(ds, v))
  81             else:
  82                 env_list.append(os.path.join(con.THT_ENV_DIR, v))
  83         elif isinstance(v, tuple):
  84                 if ds[k] == v[0]:
  85                     env_list.append(os.path.join(con.THT_ENV_DIR, v[1]))
  86     if len(env_list) == 0:
  87         try:
  88             env_list.append(os.path.join(
  89                 con.THT_ENV_DIR, sdn_map['default']))
  90         except KeyError:
  91             logging.warning("Unable to find default file for SDN")
  92
  93     return env_list
  94
  95
  96 def create_deploy_cmd(ds, ns, inv, tmp_dir,
  97                       virtual, env_file='opnfv-environment.yaml'):
  98
  99     logging.info("Creating deployment command")
 100     deploy_options = [env_file, 'network-environment.yaml']
 101     ds_opts = ds['deploy_options']
 102     deploy_options += build_sdn_env_list(ds_opts, SDN_FILE_MAP)
 103
 104     for k, v in OTHER_FILE_MAP.items():
 105         if k in ds_opts and ds_opts[k]:
 106             deploy_options.append(os.path.join(con.THT_ENV_DIR, v))
 107
 108     if ds_opts['ceph']:
 109         prep_storage_env(ds, tmp_dir)
 110         deploy_options.append(os.path.join(con.THT_ENV_DIR,
 111                                            'storage-environment.yaml'))
 112     if ds['global_params']['ha_enabled']:
 113         deploy_options.append(os.path.join(con.THT_ENV_DIR,
 114                                            'puppet-pacemaker.yaml'))
 115
 116     if virtual:
 117         deploy_options.append('virtual-environment.yaml')
 118     else:
 119         deploy_options.append('baremetal-environment.yaml')
 120
 121     num_control, num_compute = inv.get_node_counts()
 122     if num_control == 0 or num_compute == 0:
 123         logging.error("Detected 0 control or compute nodes.  Control nodes: "
 124                       "{}, compute nodes{}".format(num_control, num_compute))
 125         raise ApexDeployException("Invalid number of control or computes")
 126     elif num_control > 1 and not ds['global_params']['ha_enabled']:
 127         num_control = 1
 128     cmd = "openstack overcloud deploy --templates --timeout {} " \
 129           "--libvirt-type kvm".format(con.DEPLOY_TIMEOUT)
 130     # build cmd env args
 131     for option in deploy_options:
 132         cmd += " -e {}".format(option)
 133     cmd += " --ntp-server {}".format(ns['ntp'][0])
 134     cmd += " --control-scale {}".format(num_control)
 135     cmd += " --compute-scale {}".format(num_compute)
 136     cmd += ' --control-flavor control --compute-flavor compute'
 137     logging.info("Deploy command set: {}".format(cmd))
 138
 139     with open(os.path.join(tmp_dir, 'deploy_command'), 'w') as fh:
 140         fh.write(cmd)
 141     return cmd
 142
 143
 144 def prep_image(ds, img, tmp_dir, root_pw=None):
 145     """
 146     Locates sdn image and preps for deployment.
 147     :param ds: deploy settings
 148     :param img: sdn image
 149     :param tmp_dir: dir to store modified sdn image
 150     :param root_pw: password to configure for overcloud image
 151     :return: None
 152     """
 153     # TODO(trozet): Come up with a better way to organize this logic in this
 154     # function
 155     logging.info("Preparing image: {} for deployment".format(img))
 156     if not os.path.isfile(img):
 157         logging.error("Missing SDN image {}".format(img))
 158         raise ApexDeployException("Missing SDN image file: {}".format(img))
 159
 160     ds_opts = ds['deploy_options']
 161     virt_cmds = list()
 162     sdn = ds_opts['sdn_controller']
 163     # we need this due to rhbz #1436021
 164     # fixed in systemd-219-37.el7
 165     if sdn is not False:
 166         logging.info("Neutron openvswitch-agent disabled")
 167         virt_cmds.extend([{
 168             con.VIRT_RUN_CMD:
 169                 "rm -f /etc/systemd/system/multi-user.target.wants/"
 170                 "neutron-openvswitch-agent.service"},
 171             {
 172             con.VIRT_RUN_CMD:
 173                 "rm -f /usr/lib/systemd/system/neutron-openvswitch-agent"
 174                 ".service"
 175         }])
 176
 177     if ds_opts['vpn']:
 178         virt_cmds.append({con.VIRT_RUN_CMD: "systemctl enable zrpcd"})
 179         logging.info("ZRPC and Quagga enabled")
 180
 181     dataplane = ds_opts['dataplane']
 182     if dataplane == 'ovs_dpdk' or dataplane == 'fdio':
 183         logging.info("Enabling kernel modules for dpdk")
 184         # file to module mapping
 185         uio_types = {
 186             os.path.join(tmp_dir, 'vfio_pci.modules'): 'vfio_pci',
 187             os.path.join(tmp_dir, 'uio_pci_generic.modules'): 'uio_pci_generic'
 188         }
 189         for mod_file, mod in uio_types.items():
 190             with open(mod_file, 'w') as fh:
 191                 fh.write('#!/bin/bash\n')
 192                 fh.write('exec /sbin/modprobe {}'.format(mod))
 193                 fh.close()
 194
 195             virt_cmds.extend([
 196                 {con.VIRT_UPLOAD: "{}:/etc/sysconfig/modules/".format(
 197                     mod_file)},
 198                 {con.VIRT_RUN_CMD: "chmod 0755 /etc/sysconfig/modules/"
 199                                    "{}".format(os.path.basename(mod_file))}
 200             ])
 201     if root_pw:
 202         pw_op = "password:{}".format(root_pw)
 203         virt_cmds.append({con.VIRT_PW: pw_op})
 204     if ds_opts['sfc'] and dataplane == 'ovs':
 205         virt_cmds.extend([
 206             {con.VIRT_RUN_CMD: "yum -y install "
 207                                "/root/ovs/rpm/rpmbuild/RPMS/x86_64/"
 208                                "{}".format(OVS_NSH_KMOD_RPM)},
 209             {con.VIRT_RUN_CMD: "yum downgrade -y "
 210                                "/root/ovs/rpm/rpmbuild/RPMS/x86_64/"
 211                                "{}".format(OVS_NSH_RPM)}
 212         ])
 213     if dataplane == 'fdio':
 214         # Patch neutron with using OVS external interface for router
 215         # and add generic linux NS interface driver
 216         virt_cmds.append(
 217             {con.VIRT_RUN_CMD: "cd /usr/lib/python2.7/site-packages && patch "
 218                                "-p1 < neutron-patch-NSDriver.patch"})
 219         if sdn is False:
 220             virt_cmds.extend([
 221                 {con.VIRT_RUN_CMD: "yum remove -y vpp-lib"},
 222                 {con.VIRT_RUN_CMD: "yum install -y "
 223                                    "/root/nosdn_vpp_rpms/*.rpm"}
 224             ])
 225
 226     if sdn == 'opendaylight':
 227         if ds_opts['odl_version'] != con.DEFAULT_ODL_VERSION:
 228             virt_cmds.extend([
 229                 {con.VIRT_RUN_CMD: "yum -y remove opendaylight"},
 230                 {con.VIRT_RUN_CMD: "rm -rf /etc/puppet/modules/opendaylight"},
 231                 {con.VIRT_RUN_CMD: "cd /etc/puppet/modules && tar xzf "
 232                                    "/root/puppet-opendaylight-"
 233                                    "{}.tar.gz".format(ds_opts['odl_version'])}
 234             ])
 235             if ds_opts['odl_version'] == 'master':
 236                 virt_cmds.extend([
 237                     {con.VIRT_RUN_CMD: "rpm -ivh --nodeps /root/{}/*".format(
 238                         ds_opts['odl_version'])}
 239                 ])
 240             else:
 241                 virt_cmds.extend([
 242                     {con.VIRT_RUN_CMD: "yum -y install /root/{}/*".format(
 243                         ds_opts['odl_version'])}
 244                 ])
 245
 246         elif sdn == 'opendaylight' and 'odl_vpp_netvirt' in ds_opts \
 247                 and ds_opts['odl_vpp_netvirt']:
 248             virt_cmds.extend([
 249                 {con.VIRT_RUN_CMD: "yum -y remove opendaylight"},
 250                 {con.VIRT_RUN_CMD: "yum -y install /root/{}/*".format(
 251                     ODL_NETVIRT_VPP_RPM)}
 252             ])
 253
 254     if sdn == 'ovn':
 255         virt_cmds.extend([
 256             {con.VIRT_RUN_CMD: "cd /root/ovs28 && yum update -y "
 257                                "*openvswitch*"},
 258             {con.VIRT_RUN_CMD: "cd /root/ovs28 && yum downgrade -y "
 259                                "*openvswitch*"}
 260         ])
 261
 262     tmp_oc_image = os.path.join(tmp_dir, 'overcloud-full.qcow2')
 263     shutil.copyfile(img, tmp_oc_image)
 264     logging.debug("Temporary overcloud image stored as: {}".format(
 265         tmp_oc_image))
 266     virt_utils.virt_customize(virt_cmds, tmp_oc_image)
 267     logging.info("Overcloud image customization complete")
 268
 269
 270 def make_ssh_key():
 271     """
 272     Creates public and private ssh keys with 1024 bit RSA encryption
 273     :return: private, public key
 274     """
 275     key = rsa.generate_private_key(
 276         backend=crypto_default_backend(),
 277         public_exponent=65537,
 278         key_size=1024
 279     )
 280
 281     private_key = key.private_bytes(
 282         crypto_serialization.Encoding.PEM,
 283         crypto_serialization.PrivateFormat.PKCS8,
 284         crypto_serialization.NoEncryption())
 285     public_key = key.public_key().public_bytes(
 286         crypto_serialization.Encoding.OpenSSH,
 287         crypto_serialization.PublicFormat.OpenSSH
 288     )
 289     return private_key.decode('utf-8'), public_key.decode('utf-8')
 290
 291
 292 def prep_env(ds, ns, inv, opnfv_env, net_env, tmp_dir):
 293     """
 294     Creates modified opnfv/network environments for deployment
 295     :param ds: deploy settings
 296     :param ns: network settings
 297     :param inv: node inventory
 298     :param opnfv_env: file path for opnfv-environment file
 299     :param net_env: file path for network-environment file
 300     :param tmp_dir: Apex tmp dir
 301     :return:
 302     """
 303
 304     logging.info("Preparing opnfv-environment and network-environment files")
 305     ds_opts = ds['deploy_options']
 306     tmp_opnfv_env = os.path.join(tmp_dir, os.path.basename(opnfv_env))
 307     shutil.copyfile(opnfv_env, tmp_opnfv_env)
 308     tenant_nic_map = ns['networks']['tenant']['nic_mapping']
 309     tenant_ctrl_nic = tenant_nic_map['controller']['members'][0]
 310     tenant_comp_nic = tenant_nic_map['compute']['members'][0]
 311
 312     # SSH keys
 313     private_key, public_key = make_ssh_key()
 314
 315     # Make easier/faster variables to index in the file editor
 316     if 'performance' in ds_opts:
 317         perf = True
 318         # vpp
 319         if 'vpp' in ds_opts['performance']['Compute']:
 320             perf_vpp_comp = ds_opts['performance']['Compute']['vpp']
 321         else:
 322             perf_vpp_comp = None
 323         if 'vpp' in ds_opts['performance']['Controller']:
 324             perf_vpp_ctrl = ds_opts['performance']['Controller']['vpp']
 325         else:
 326             perf_vpp_ctrl = None
 327
 328         # ovs
 329         if 'ovs' in ds_opts['performance']['Compute']:
 330             perf_ovs_comp = ds_opts['performance']['Compute']['ovs']
 331         else:
 332             perf_ovs_comp = None
 333
 334         # kernel
 335         if 'kernel' in ds_opts['performance']['Compute']:
 336             perf_kern_comp = ds_opts['performance']['Compute']['kernel']
 337         else:
 338             perf_kern_comp = None
 339     else:
 340         perf = False
 341
 342     # Modify OPNFV environment
 343     # TODO: Change to build a dict and outputing yaml rather than parsing
 344     for line in fileinput.input(tmp_opnfv_env, inplace=True):
 345         line = line.strip('\n')
 346         output_line = line
 347         if 'CloudDomain' in line:
 348             output_line = "  CloudDomain: {}".format(ns['domain_name'])
 349         elif 'replace_private_key' in line:
 350             output_line = "    private_key: |\n"
 351             key_out = ''
 352             for line in private_key.splitlines():
 353                 key_out += "      {}\n".format(line)
 354             output_line += key_out
 355         elif 'replace_public_key' in line:
 356             output_line = "    public_key: '{}'".format(public_key)
 357
 358         if ds_opts['sdn_controller'] == 'opendaylight' and \
 359                 'odl_vpp_routing_node' in ds_opts:
 360             if 'opendaylight::vpp_routing_node' in line:
 361                 output_line = ("    opendaylight::vpp_routing_node: {}.{}"
 362                                .format(ds_opts['odl_vpp_routing_node'],
 363                                        ns['domain_name']))
 364             elif 'ControllerExtraConfig' in line:
 365                 output_line = ("  ControllerExtraConfig:\n    "
 366                                "tripleo::profile::base::neutron::agents::"
 367                                "honeycomb::interface_role_mapping:"
 368                                " ['{}:tenant-interface]'"
 369                                .format(tenant_ctrl_nic))
 370             elif 'NovaComputeExtraConfig' in line:
 371                 output_line = ("  NovaComputeExtraConfig:\n    "
 372                                "tripleo::profile::base::neutron::agents::"
 373                                "honeycomb::interface_role_mapping:"
 374                                " ['{}:tenant-interface]'"
 375                                .format(tenant_comp_nic))
 376         elif not ds_opts['sdn_controller'] and ds_opts['dataplane'] == 'fdio':
 377             if 'NeutronVPPAgentPhysnets' in line:
 378                 output_line = ("  NeutronVPPAgentPhysnets: 'datacentre:{}'".
 379                                format(tenant_ctrl_nic))
 380         elif ds_opts['sdn_controller'] == 'opendaylight' and ds_opts.get(
 381                 'dvr') is True:
 382             if 'OS::TripleO::Services::NeutronDhcpAgent' in line:
 383                 output_line = ''
 384             elif 'NeutronDhcpAgentsPerNetwork' in line:
 385                 num_control, num_compute = inv.get_node_counts()
 386                 output_line = ("  NeutronDhcpAgentsPerNetwork: {}"
 387                                .format(num_compute))
 388             elif 'ComputeServices' in line:
 389                 output_line = ("  ComputeServices:\n"
 390                                "    - OS::TripleO::Services::NeutronDhcpAgent")
 391
 392         if perf:
 393             for role in 'NovaCompute', 'Controller':
 394                 if role == 'NovaCompute':
 395                     perf_opts = perf_vpp_comp
 396                 else:
 397                     perf_opts = perf_vpp_ctrl
 398                 cfg = "{}ExtraConfig".format(role)
 399                 if cfg in line and perf_opts:
 400                     perf_line = ''
 401                     if 'main-core' in perf_opts:
 402                         perf_line += ("\n    fdio::vpp_cpu_main_core: '{}'"
 403                                       .format(perf_opts['main-core']))
 404                     if 'corelist-workers' in perf_opts:
 405                         perf_line += ("\n    "
 406                                       "fdio::vpp_cpu_corelist_workers: '{}'"
 407                                       .format(perf_opts['corelist-workers']))
 408                     if perf_line:
 409                         output_line = ("  {}:{}".format(cfg, perf_line))
 410
 411             # kernel args
 412             # (FIXME) use compute's kernel settings for all nodes for now.
 413             if 'ComputeKernelArgs' in line and perf_kern_comp:
 414                 kernel_args = ''
 415                 for k, v in perf_kern_comp.items():
 416                     kernel_args += "{}={} ".format(k, v)
 417                 if kernel_args:
 418                     output_line = "  ComputeKernelArgs: '{}'".\
 419                         format(kernel_args)
 420             if ds_opts['dataplane'] == 'ovs_dpdk' and perf_ovs_comp:
 421                 for k, v in OVS_PERF_MAP.items():
 422                     if k in line and v in perf_ovs_comp:
 423                         output_line = "  {}: '{}'".format(k, perf_ovs_comp[v])
 424
 425         print(output_line)
 426
 427     logging.info("opnfv-environment file written to {}".format(tmp_opnfv_env))
 428
 429     # Modify Network environment
 430     for line in fileinput.input(net_env, inplace=True):
 431         line = line.strip('\n')
 432         if 'ComputeExtraConfigPre' in line and \
 433                 ds_opts['dataplane'] == 'ovs_dpdk':
 434             print('  OS::TripleO::ComputeExtraConfigPre: '
 435                   './ovs-dpdk-preconfig.yaml')
 436         elif ((perf and perf_kern_comp) or ds_opts.get('rt_kvm')) and \
 437                 'resource_registry' in line:
 438             print("resource_registry:\n"
 439                   "  OS::TripleO::NodeUserData: first-boot.yaml")
 440         elif perf and perf_kern_comp and \
 441                 'NovaSchedulerDefaultFilters' in line:
 442             print("  NovaSchedulerDefaultFilters: 'RamFilter,"
 443                   "ComputeFilter,AvailabilityZoneFilter,"
 444                   "ComputeCapabilitiesFilter,ImagePropertiesFilter,"
 445                   "NUMATopologyFilter'")
 446         else:
 447             print(line)
 448
 449     logging.info("network-environment file written to {}".format(net_env))
 450
 451
 452 def generate_ceph_key():
 453     key = os.urandom(16)
 454     header = struct.pack('<hiih', 1, int(time.time()), 0, len(key))
 455     return base64.b64encode(header + key)
 456
 457
 458 def prep_storage_env(ds, tmp_dir):
 459     """
 460     Creates storage environment file for deployment.  Source file is copied by
 461     undercloud playbook to host.
 462     :param ds:
 463     :param tmp_dir:
 464     :return:
 465     """
 466     ds_opts = ds['deploy_options']
 467     storage_file = os.path.join(tmp_dir, 'storage-environment.yaml')
 468     if not os.path.isfile(storage_file):
 469         logging.error("storage-environment file is not in tmp directory: {}. "
 470                       "Check if file was copied from "
 471                       "undercloud".format(tmp_dir))
 472         raise ApexDeployException("storage-environment file not copied from "
 473                                   "undercloud")
 474     for line in fileinput.input(storage_file, inplace=True):
 475         line = line.strip('\n')
 476         if 'CephClusterFSID' in line:
 477             print("  CephClusterFSID: {}".format(str(uuid.uuid4())))
 478         elif 'CephMonKey' in line:
 479             print("  CephMonKey: {}".format(generate_ceph_key().decode(
 480                 'utf-8')))
 481         elif 'CephAdminKey' in line:
 482             print("  CephAdminKey: {}".format(generate_ceph_key().decode(
 483                 'utf-8')))
 484         else:
 485             print(line)
 486     if 'ceph_device' in ds_opts and ds_opts['ceph_device']:
 487         with open(storage_file, 'a') as fh:
 488             fh.write('  ExtraConfig:\n')
 489             fh.write("    ceph::profile::params::osds:{{{}:{{}}}}\n".format(
 490                 ds_opts['ceph_device']
 491             ))
 492
 493
 494 def external_network_cmds(ns):
 495     """
 496     Generates external network openstack commands
 497     :param ns: network settings
 498     :return: list of commands to configure external network
 499     """
 500     if 'external' in ns.enabled_network_list:
 501         net_config = ns['networks']['external'][0]
 502         external = True
 503         pool_start, pool_end = net_config['floating_ip_range']
 504     else:
 505         net_config = ns['networks']['admin']
 506         external = False
 507         pool_start, pool_end = ns['apex']['networks']['admin'][
 508             'introspection_range']
 509     nic_config = net_config['nic_mapping']
 510     gateway = net_config['gateway']
 511     cmds = list()
 512     # create network command
 513     if nic_config['compute']['vlan'] == 'native':
 514         ext_type = 'flat'
 515     else:
 516         ext_type = "vlan --provider-segment {}".format(nic_config[
 517                                                        'compute']['vlan'])
 518     cmds.append("openstack network create external --project service "
 519                 "--external --provider-network-type {} "
 520                 "--provider-physical-network datacentre".format(ext_type))
 521     # create subnet command
 522     cidr = net_config['cidr']
 523     subnet_cmd = "openstack subnet create external-subnet --project " \
 524                  "service --network external --no-dhcp --gateway {} " \
 525                  "--allocation-pool start={},end={} --subnet-range " \
 526                  "{}".format(gateway, pool_start, pool_end, str(cidr))
 527     if external and cidr.version == 6:
 528         subnet_cmd += ' --ip-version 6 --ipv6-ra-mode slaac ' \
 529                       '--ipv6-address-mode slaac'
 530     cmds.append(subnet_cmd)
 531     logging.debug("Neutron external network commands determined "
 532                   "as: {}".format(cmds))
 533     return cmds
 534
 535
 536 def create_congress_cmds(overcloud_file):
 537     drivers = ['nova', 'neutronv2', 'cinder', 'glancev2', 'keystone', 'doctor']
 538     overcloudrc = parsers.parse_overcloudrc(overcloud_file)
 539     logging.info("Creating congress commands")
 540     try:
 541         ds_cfg = [
 542             "username={}".format(overcloudrc['OS_USERNAME']),
 543             "tenant_name={}".format(overcloudrc['OS_PROJECT_NAME']),
 544             "password={}".format(overcloudrc['OS_PASSWORD']),
 545             "auth_url={}".format(overcloudrc['OS_AUTH_URL'])
 546         ]
 547     except KeyError:
 548         logging.error("Unable to find all keys required for congress in "
 549                       "overcloudrc: OS_USERNAME, OS_PROJECT_NAME, "
 550                       "OS_PASSWORD, OS_AUTH_URL.  Please check overcloudrc "
 551                       "file: {}".format(overcloud_file))
 552         raise
 553     cmds = list()
 554     ds_cfg = '--config ' + ' --config '.join(ds_cfg)
 555
 556     for driver in drivers:
 557         if driver == 'doctor':
 558             cmd = "{} \"{}\"".format(driver, driver)
 559         else:
 560             cmd = "{} \"{}\" {}".format(driver, driver, ds_cfg)
 561         if driver == 'nova':
 562             cmd += ' --config api_version="2.34"'
 563         logging.debug("Congress command created: {}".format(cmd))
 564         cmds.append(cmd)
 565     return cmds