apex/overcloud/deploy.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 import base64
  11 import fileinput
  12 import logging
  13 import os
  14 import re
  15 import shutil
  16 import uuid
  17 import struct
  18 import time
  19
  20 from apex.common import constants as con
  21 from apex.common.exceptions import ApexDeployException
  22 from apex.common import parsers
  23 from apex.virtual import utils as virt_utils
  24 from cryptography.hazmat.primitives import serialization as \
  25     crypto_serialization
  26 from cryptography.hazmat.primitives.asymmetric import rsa
  27 from cryptography.hazmat.backends import default_backend as \
  28     crypto_default_backend
  29
  30
  31 SDN_FILE_MAP = {
  32     'opendaylight': {
  33         'sfc': 'neutron-sfc-opendaylight.yaml',
  34         'vpn': 'neutron-bgpvpn-opendaylight.yaml',
  35         'gluon': 'gluon.yaml',
  36         'vpp': {
  37             'odl_vpp_netvirt': 'neutron-opendaylight-netvirt-vpp.yaml',
  38             'dvr': 'neutron-opendaylight-fdio-dvr.yaml',
  39             'default': 'neutron-opendaylight-honeycomb.yaml'
  40         },
  41         'default': 'neutron-opendaylight.yaml',
  42     },
  43     'onos': {
  44         'sfc': 'neutron-onos-sfc.yaml',
  45         'default': 'neutron-onos.yaml'
  46     },
  47     'ovn': 'neutron-ml2-ovn.yaml',
  48     False: {
  49         'vpp': 'neutron-ml2-vpp.yaml',
  50         'dataplane': ('ovs_dpdk', 'neutron-ovs-dpdk.yaml')
  51     }
  52 }
  53
  54 OTHER_FILE_MAP = {
  55     'tacker': 'enable_tacker.yaml',
  56     'congress': 'enable_congress.yaml',
  57     'barometer': 'enable_barometer.yaml',
  58     'rt_kvm': 'enable_rt_kvm.yaml'
  59 }
  60
  61 OVS_PERF_MAP = {
  62     'HostCpusList': 'dpdk_cores',
  63     'NeutronDpdkCoreList': 'pmd_cores',
  64     'NeutronDpdkSocketMemory': 'socket_memory',
  65     'NeutronDpdkMemoryChannels': 'memory_channels'
  66 }
  67
  68 OVS_NSH_KMOD_RPM = "openvswitch-kmod-2.6.1-1.el7.centos.x86_64.rpm"
  69 OVS_NSH_RPM = "openvswitch-2.6.1-1.el7.centos.x86_64.rpm"
  70 ODL_NETVIRT_VPP_RPM = "/root/opendaylight-7.0.0-0.1.20170531snap665.el7" \
  71                       ".noarch.rpm"
  72
  73
  74 def build_sdn_env_list(ds, sdn_map, env_list=None):
  75     if env_list is None:
  76         env_list = list()
  77     for k, v in sdn_map.items():
  78         if ds['sdn_controller'] == k or (k in ds and ds[k] is True):
  79             if isinstance(v, dict):
  80                 env_list.extend(build_sdn_env_list(ds, v))
  81             else:
  82                 env_list.append(os.path.join(con.THT_ENV_DIR, v))
  83         elif isinstance(v, tuple):
  84                 if ds[k] == v[0]:
  85                     env_list.append(os.path.join(con.THT_ENV_DIR, v[1]))
  86     if len(env_list) == 0:
  87         try:
  88             env_list.append(os.path.join(
  89                 con.THT_ENV_DIR, sdn_map['default']))
  90         except KeyError:
  91             logging.warning("Unable to find default file for SDN")
  92
  93     return env_list
  94
  95
  96 def create_deploy_cmd(ds, ns, inv, tmp_dir,
  97                       virtual, env_file='opnfv-environment.yaml'):
  98
  99     logging.info("Creating deployment command")
 100     deploy_options = [env_file, 'network-environment.yaml']
 101     ds_opts = ds['deploy_options']
 102     deploy_options += build_sdn_env_list(ds_opts, SDN_FILE_MAP)
 103
 104     for k, v in OTHER_FILE_MAP.items():
 105         if k in ds_opts and ds_opts[k]:
 106             deploy_options.append(os.path.join(con.THT_ENV_DIR, v))
 107
 108     if ds_opts['ceph']:
 109         prep_storage_env(ds, tmp_dir)
 110         deploy_options.append(os.path.join(con.THT_ENV_DIR,
 111                                            'storage-environment.yaml'))
 112     if ds['global_params']['ha_enabled']:
 113         deploy_options.append(os.path.join(con.THT_ENV_DIR,
 114                                            'puppet-pacemaker.yaml'))
 115
 116     if virtual:
 117         deploy_options.append('virtual-environment.yaml')
 118     else:
 119         deploy_options.append('baremetal-environment.yaml')
 120
 121     num_control, num_compute = inv.get_node_counts()
 122     if num_control == 0 or num_compute == 0:
 123         logging.error("Detected 0 control or compute nodes.  Control nodes: "
 124                       "{}, compute nodes{}".format(num_control, num_compute))
 125         raise ApexDeployException("Invalid number of control or computes")
 126     elif num_control > 1 and not ds['global_params']['ha_enabled']:
 127         num_control = 1
 128     cmd = "openstack overcloud deploy --templates --timeout {} " \
 129           "--libvirt-type kvm".format(con.DEPLOY_TIMEOUT)
 130     # build cmd env args
 131     for option in deploy_options:
 132         cmd += " -e {}".format(option)
 133     cmd += " --ntp-server {}".format(ns['ntp'][0])
 134     cmd += " --control-scale {}".format(num_control)
 135     cmd += " --compute-scale {}".format(num_compute)
 136     cmd += ' --control-flavor control --compute-flavor compute'
 137     logging.info("Deploy command set: {}".format(cmd))
 138
 139     with open(os.path.join(tmp_dir, 'deploy_command'), 'w') as fh:
 140         fh.write(cmd)
 141     return cmd
 142
 143
 144 def prep_image(ds, img, tmp_dir, root_pw=None):
 145     """
 146     Locates sdn image and preps for deployment.
 147     :param ds: deploy settings
 148     :param img: sdn image
 149     :param tmp_dir: dir to store modified sdn image
 150     :param root_pw: password to configure for overcloud image
 151     :return: None
 152     """
 153     # TODO(trozet): Come up with a better way to organize this logic in this
 154     # function
 155     logging.info("Preparing image: {} for deployment".format(img))
 156     if not os.path.isfile(img):
 157         logging.error("Missing SDN image {}".format(img))
 158         raise ApexDeployException("Missing SDN image file: {}".format(img))
 159
 160     ds_opts = ds['deploy_options']
 161     virt_cmds = list()
 162     sdn = ds_opts['sdn_controller']
 163     # we need this due to rhbz #1436021
 164     # fixed in systemd-219-37.el7
 165     if sdn is not False:
 166         logging.info("Neutron openvswitch-agent disabled")
 167         virt_cmds.extend([{
 168             con.VIRT_RUN_CMD:
 169                 "rm -f /etc/systemd/system/multi-user.target.wants/"
 170                 "neutron-openvswitch-agent.service"},
 171             {
 172             con.VIRT_RUN_CMD:
 173                 "rm -f /usr/lib/systemd/system/neutron-openvswitch-agent"
 174                 ".service"
 175         }])
 176
 177     if ds_opts['vpn']:
 178         virt_cmds.append({con.VIRT_RUN_CMD: "systemctl enable zrpcd"})
 179         logging.info("ZRPC and Quagga enabled")
 180
 181     dataplane = ds_opts['dataplane']
 182     if dataplane == 'ovs_dpdk' or dataplane == 'fdio':
 183         logging.info("Enabling kernel modules for dpdk")
 184         # file to module mapping
 185         uio_types = {
 186             os.path.join(tmp_dir, 'vfio_pci.modules'): 'vfio_pci',
 187             os.path.join(tmp_dir, 'uio_pci_generic.modules'): 'uio_pci_generic'
 188         }
 189         for mod_file, mod in uio_types.items():
 190             with open(mod_file, 'w') as fh:
 191                 fh.write('#!/bin/bash\n')
 192                 fh.write('exec /sbin/modprobe {}'.format(mod))
 193                 fh.close()
 194
 195             virt_cmds.extend([
 196                 {con.VIRT_UPLOAD: "{}:/etc/sysconfig/modules/".format(
 197                     mod_file)},
 198                 {con.VIRT_RUN_CMD: "chmod 0755 /etc/sysconfig/modules/"
 199                                    "{}".format(os.path.basename(mod_file))}
 200             ])
 201     if root_pw:
 202         pw_op = "password:{}".format(root_pw)
 203         virt_cmds.append({con.VIRT_PW: pw_op})
 204     if ds_opts['sfc'] and dataplane == 'ovs':
 205         virt_cmds.extend([
 206             {con.VIRT_RUN_CMD: "yum -y install "
 207                                "/root/ovs/rpm/rpmbuild/RPMS/x86_64/"
 208                                "{}".format(OVS_NSH_KMOD_RPM)},
 209             {con.VIRT_RUN_CMD: "yum downgrade -y "
 210                                "/root/ovs/rpm/rpmbuild/RPMS/x86_64/"
 211                                "{}".format(OVS_NSH_RPM)}
 212         ])
 213     if dataplane == 'fdio':
 214         # Patch neutron with using OVS external interface for router
 215         # and add generic linux NS interface driver
 216         virt_cmds.append(
 217             {con.VIRT_RUN_CMD: "cd /usr/lib/python2.7/site-packages && patch "
 218                                "-p1 < neutron-patch-NSDriver.patch"})
 219         if sdn is False:
 220             virt_cmds.extend([
 221                 {con.VIRT_RUN_CMD: "yum remove -y vpp-lib"},
 222                 {con.VIRT_RUN_CMD: "yum install -y "
 223                                    "/root/nosdn_vpp_rpms/*.rpm"}
 224             ])
 225
 226     if sdn == 'opendaylight':
 227         if ds_opts['odl_version'] != con.DEFAULT_ODL_VERSION:
 228             virt_cmds.extend([
 229                 {con.VIRT_RUN_CMD: "yum -y remove opendaylight"},
 230                 {con.VIRT_RUN_CMD: "rm -rf /etc/puppet/modules/opendaylight"},
 231                 {con.VIRT_RUN_CMD: "cd /etc/puppet/modules && tar xzf "
 232                                    "/root/puppet-opendaylight-"
 233                                    "{}.tar.gz".format(ds_opts['odl_version'])}
 234             ])
 235             if ds_opts['odl_version'] == 'master':
 236                 virt_cmds.extend([
 237                     {con.VIRT_RUN_CMD: "rpm -ivh --nodeps /root/{}/*".format(
 238                         ds_opts['odl_version'])}
 239                 ])
 240             else:
 241                 virt_cmds.extend([
 242                     {con.VIRT_RUN_CMD: "yum -y install /root/{}/*".format(
 243                         ds_opts['odl_version'])}
 244                 ])
 245
 246         elif sdn == 'opendaylight' and 'odl_vpp_netvirt' in ds_opts \
 247                 and ds_opts['odl_vpp_netvirt']:
 248             virt_cmds.extend([
 249                 {con.VIRT_RUN_CMD: "yum -y remove opendaylight"},
 250                 {con.VIRT_RUN_CMD: "yum -y install /root/{}/*".format(
 251                     ODL_NETVIRT_VPP_RPM)}
 252             ])
 253
 254     if sdn == 'ovn':
 255         virt_cmds.extend([
 256             {con.VIRT_RUN_CMD: "cd /root/ovs28 && yum update -y "
 257                                "*openvswitch*"},
 258             {con.VIRT_RUN_CMD: "cd /root/ovs28 && yum downgrade -y "
 259                                "*openvswitch*"}
 260         ])
 261
 262     tmp_oc_image = os.path.join(tmp_dir, 'overcloud-full.qcow2')
 263     shutil.copyfile(img, tmp_oc_image)
 264     logging.debug("Temporary overcloud image stored as: {}".format(
 265         tmp_oc_image))
 266     virt_utils.virt_customize(virt_cmds, tmp_oc_image)
 267     logging.info("Overcloud image customization complete")
 268
 269
 270 def make_ssh_key():
 271     """
 272     Creates public and private ssh keys with 1024 bit RSA encryption
 273     :return: private, public key
 274     """
 275     key = rsa.generate_private_key(
 276         backend=crypto_default_backend(),
 277         public_exponent=65537,
 278         key_size=1024
 279     )
 280
 281     private_key = key.private_bytes(
 282         crypto_serialization.Encoding.PEM,
 283         crypto_serialization.PrivateFormat.PKCS8,
 284         crypto_serialization.NoEncryption())
 285     public_key = key.public_key().public_bytes(
 286         crypto_serialization.Encoding.OpenSSH,
 287         crypto_serialization.PublicFormat.OpenSSH
 288     )
 289     return private_key.decode('utf-8'), public_key.decode('utf-8')
 290
 291
 292 def prep_env(ds, ns, inv, opnfv_env, net_env, tmp_dir):
 293     """
 294     Creates modified opnfv/network environments for deployment
 295     :param ds: deploy settings
 296     :param ns: network settings
 297     :param inv: node inventory
 298     :param opnfv_env: file path for opnfv-environment file
 299     :param net_env: file path for network-environment file
 300     :param tmp_dir: Apex tmp dir
 301     :return:
 302     """
 303
 304     logging.info("Preparing opnfv-environment and network-environment files")
 305     ds_opts = ds['deploy_options']
 306     tmp_opnfv_env = os.path.join(tmp_dir, os.path.basename(opnfv_env))
 307     shutil.copyfile(opnfv_env, tmp_opnfv_env)
 308     tenant_nic_map = ns['networks']['tenant']['nic_mapping']
 309     tenant_nic = dict()
 310     tenant_nic['Controller'] = tenant_nic_map['controller']['members'][0]
 311     tenant_nic['NovaCompute'] = tenant_nic_map['compute']['members'][0]
 312     external_nic_map = ns['networks']['external'][0]['nic_mapping']
 313     external_nic = dict()
 314     external_nic['NovaCompute'] = external_nic_map['compute']['members'][0]
 315
 316     # SSH keys
 317     private_key, public_key = make_ssh_key()
 318
 319     # Make easier/faster variables to index in the file editor
 320     if 'performance' in ds_opts:
 321         perf = True
 322         # vpp
 323         if 'vpp' in ds_opts['performance']['Compute']:
 324             perf_vpp_comp = ds_opts['performance']['Compute']['vpp']
 325         else:
 326             perf_vpp_comp = None
 327         if 'vpp' in ds_opts['performance']['Controller']:
 328             perf_vpp_ctrl = ds_opts['performance']['Controller']['vpp']
 329         else:
 330             perf_vpp_ctrl = None
 331
 332         # ovs
 333         if 'ovs' in ds_opts['performance']['Compute']:
 334             perf_ovs_comp = ds_opts['performance']['Compute']['ovs']
 335         else:
 336             perf_ovs_comp = None
 337
 338         # kernel
 339         if 'kernel' in ds_opts['performance']['Compute']:
 340             perf_kern_comp = ds_opts['performance']['Compute']['kernel']
 341         else:
 342             perf_kern_comp = None
 343     else:
 344         perf = False
 345
 346     # Modify OPNFV environment
 347     # TODO: Change to build a dict and outputing yaml rather than parsing
 348     for line in fileinput.input(tmp_opnfv_env, inplace=True):
 349         line = line.strip('\n')
 350         output_line = line
 351         if 'CloudDomain' in line:
 352             output_line = "  CloudDomain: {}".format(ns['domain_name'])
 353         elif 'replace_private_key' in line:
 354             output_line = "    private_key: |\n"
 355             key_out = ''
 356             for line in private_key.splitlines():
 357                 key_out += "      {}\n".format(line)
 358             output_line += key_out
 359         elif 'replace_public_key' in line:
 360             output_line = "    public_key: '{}'".format(public_key)
 361
 362         if ds_opts['sdn_controller'] == 'opendaylight' and \
 363                 'odl_vpp_routing_node' in ds_opts:
 364             if 'opendaylight::vpp_routing_node' in line:
 365                 output_line = ("    opendaylight::vpp_routing_node: {}.{}"
 366                                .format(ds_opts['odl_vpp_routing_node'],
 367                                        ns['domain_name']))
 368         elif not ds_opts['sdn_controller'] and ds_opts['dataplane'] == 'fdio':
 369             if 'NeutronVPPAgentPhysnets' in line:
 370                 output_line = ("  NeutronVPPAgentPhysnets: 'datacentre:{}'".
 371                                format(tenant_nic['Controller']))
 372         elif ds_opts['sdn_controller'] == 'opendaylight' and ds_opts.get(
 373                 'dvr') is True:
 374             if 'OS::TripleO::Services::NeutronDhcpAgent' in line:
 375                 output_line = ''
 376             elif 'NeutronDhcpAgentsPerNetwork' in line:
 377                 num_control, num_compute = inv.get_node_counts()
 378                 output_line = ("  NeutronDhcpAgentsPerNetwork: {}"
 379                                .format(num_compute))
 380             elif 'ComputeServices' in line:
 381                 output_line = ("  ComputeServices:\n"
 382                                "    - OS::TripleO::Services::NeutronDhcpAgent")
 383
 384         if perf:
 385             for role in 'NovaCompute', 'Controller':
 386                 if role == 'NovaCompute':
 387                     perf_opts = perf_vpp_comp
 388                 else:
 389                     perf_opts = perf_vpp_ctrl
 390                 cfg = "{}ExtraConfig".format(role)
 391                 if cfg in line and perf_opts:
 392                     perf_line = ''
 393                     if 'main-core' in perf_opts:
 394                         perf_line += ("\n    fdio::vpp_cpu_main_core: '{}'"
 395                                       .format(perf_opts['main-core']))
 396                     if 'corelist-workers' in perf_opts:
 397                         perf_line += ("\n    "
 398                                       "fdio::vpp_cpu_corelist_workers: '{}'"
 399                                       .format(perf_opts['corelist-workers']))
 400                     if ds_opts['sdn_controller'] == 'opendaylight' and \
 401                             ds_opts['dataplane'] == 'fdio':
 402                         if role == 'NovaCompute':
 403                             perf_line += ("\n    "
 404                                           "tripleo::profile::base::neutron::"
 405                                           "agents::honeycomb::"
 406                                           "interface_role_mapping:"
 407                                           " ['{}:tenant-interface',"
 408                                           "'{}:public-interface']"
 409                                           .format(tenant_nic[role],
 410                                                   external_nic[role]))
 411                         else:
 412                             perf_line += ("\n    "
 413                                           "tripleo::profile::base::neutron::"
 414                                           "agents::honeycomb::"
 415                                           "interface_role_mapping:"
 416                                           " ['{}:tenant-interface']"
 417                                           .format(tenant_nic[role]))
 418                     if perf_line:
 419                         output_line = ("  {}:{}".format(cfg, perf_line))
 420
 421             # kernel args
 422             # (FIXME) use compute's kernel settings for all nodes for now.
 423             if 'ComputeKernelArgs' in line and perf_kern_comp:
 424                 kernel_args = ''
 425                 for k, v in perf_kern_comp.items():
 426                     kernel_args += "{}={} ".format(k, v)
 427                 if kernel_args:
 428                     output_line = "  ComputeKernelArgs: '{}'".\
 429                         format(kernel_args)
 430             if ds_opts['dataplane'] == 'ovs_dpdk' and perf_ovs_comp:
 431                 for k, v in OVS_PERF_MAP.items():
 432                     if k in line and v in perf_ovs_comp:
 433                         output_line = "  {}: '{}'".format(k, perf_ovs_comp[v])
 434
 435         print(output_line)
 436
 437     logging.info("opnfv-environment file written to {}".format(tmp_opnfv_env))
 438
 439     # Modify Network environment
 440     for line in fileinput.input(net_env, inplace=True):
 441         line = line.strip('\n')
 442         if 'ComputeExtraConfigPre' in line and \
 443                 ds_opts['dataplane'] == 'ovs_dpdk':
 444             print('  OS::TripleO::ComputeExtraConfigPre: '
 445                   './ovs-dpdk-preconfig.yaml')
 446         elif ((perf and perf_kern_comp) or ds_opts.get('rt_kvm')) and \
 447                 'resource_registry' in line:
 448             print("resource_registry:\n"
 449                   "  OS::TripleO::NodeUserData: first-boot.yaml")
 450         elif perf and perf_kern_comp and \
 451                 'NovaSchedulerDefaultFilters' in line:
 452             print("  NovaSchedulerDefaultFilters: 'RamFilter,"
 453                   "ComputeFilter,AvailabilityZoneFilter,"
 454                   "ComputeCapabilitiesFilter,ImagePropertiesFilter,"
 455                   "NUMATopologyFilter'")
 456         else:
 457             print(line)
 458
 459     logging.info("network-environment file written to {}".format(net_env))
 460
 461
 462 def generate_ceph_key():
 463     key = os.urandom(16)
 464     header = struct.pack('<hiih', 1, int(time.time()), 0, len(key))
 465     return base64.b64encode(header + key)
 466
 467
 468 def prep_storage_env(ds, tmp_dir):
 469     """
 470     Creates storage environment file for deployment.  Source file is copied by
 471     undercloud playbook to host.
 472     :param ds:
 473     :param tmp_dir:
 474     :return:
 475     """
 476     ds_opts = ds['deploy_options']
 477     storage_file = os.path.join(tmp_dir, 'storage-environment.yaml')
 478     if not os.path.isfile(storage_file):
 479         logging.error("storage-environment file is not in tmp directory: {}. "
 480                       "Check if file was copied from "
 481                       "undercloud".format(tmp_dir))
 482         raise ApexDeployException("storage-environment file not copied from "
 483                                   "undercloud")
 484     for line in fileinput.input(storage_file, inplace=True):
 485         line = line.strip('\n')
 486         if 'CephClusterFSID' in line:
 487             print("  CephClusterFSID: {}".format(str(uuid.uuid4())))
 488         elif 'CephMonKey' in line:
 489             print("  CephMonKey: {}".format(generate_ceph_key().decode(
 490                 'utf-8')))
 491         elif 'CephAdminKey' in line:
 492             print("  CephAdminKey: {}".format(generate_ceph_key().decode(
 493                 'utf-8')))
 494         else:
 495             print(line)
 496     if 'ceph_device' in ds_opts and ds_opts['ceph_device']:
 497         with open(storage_file, 'a') as fh:
 498             fh.write('  ExtraConfig:\n')
 499             fh.write("    ceph::profile::params::osds:{{{}:{{}}}}\n".format(
 500                 ds_opts['ceph_device']
 501             ))
 502
 503
 504 def external_network_cmds(ns):
 505     """
 506     Generates external network openstack commands
 507     :param ns: network settings
 508     :return: list of commands to configure external network
 509     """
 510     if 'external' in ns.enabled_network_list:
 511         net_config = ns['networks']['external'][0]
 512         external = True
 513         pool_start, pool_end = net_config['floating_ip_range']
 514     else:
 515         net_config = ns['networks']['admin']
 516         external = False
 517         pool_start, pool_end = ns['apex']['networks']['admin'][
 518             'introspection_range']
 519     nic_config = net_config['nic_mapping']
 520     gateway = net_config['gateway']
 521     cmds = list()
 522     # create network command
 523     if nic_config['compute']['vlan'] == 'native':
 524         ext_type = 'flat'
 525     else:
 526         ext_type = "vlan --provider-segment {}".format(nic_config[
 527                                                        'compute']['vlan'])
 528     cmds.append("openstack network create external --project service "
 529                 "--external --provider-network-type {} "
 530                 "--provider-physical-network datacentre".format(ext_type))
 531     # create subnet command
 532     cidr = net_config['cidr']
 533     subnet_cmd = "openstack subnet create external-subnet --project " \
 534                  "service --network external --no-dhcp --gateway {} " \
 535                  "--allocation-pool start={},end={} --subnet-range " \
 536                  "{}".format(gateway, pool_start, pool_end, str(cidr))
 537     if external and cidr.version == 6:
 538         subnet_cmd += ' --ip-version 6 --ipv6-ra-mode slaac ' \
 539                       '--ipv6-address-mode slaac'
 540     cmds.append(subnet_cmd)
 541     logging.debug("Neutron external network commands determined "
 542                   "as: {}".format(cmds))
 543     return cmds
 544
 545
 546 def create_congress_cmds(overcloud_file):
 547     drivers = ['nova', 'neutronv2', 'cinder', 'glancev2', 'keystone', 'doctor']
 548     overcloudrc = parsers.parse_overcloudrc(overcloud_file)
 549     logging.info("Creating congress commands")
 550     try:
 551         ds_cfg = [
 552             "username={}".format(overcloudrc['OS_USERNAME']),
 553             "tenant_name={}".format(overcloudrc['OS_PROJECT_NAME']),
 554             "password={}".format(overcloudrc['OS_PASSWORD']),
 555             "auth_url={}".format(overcloudrc['OS_AUTH_URL'])
 556         ]
 557     except KeyError:
 558         logging.error("Unable to find all keys required for congress in "
 559                       "overcloudrc: OS_USERNAME, OS_PROJECT_NAME, "
 560                       "OS_PASSWORD, OS_AUTH_URL.  Please check overcloudrc "
 561                       "file: {}".format(overcloud_file))
 562         raise
 563     cmds = list()
 564     ds_cfg = '--config ' + ' --config '.join(ds_cfg)
 565
 566     for driver in drivers:
 567         if driver == 'doctor':
 568             cmd = "{} \"{}\"".format(driver, driver)
 569         else:
 570             cmd = "{} \"{}\" {}".format(driver, driver, ds_cfg)
 571         if driver == 'nova':
 572             cmd += ' --config api_version="2.34"'
 573         logging.debug("Congress command created: {}".format(cmd))
 574         cmds.append(cmd)
 575     return cmds