build/instack.sh

   1 #!/bin/sh
   2 ##############################################################################
   3 # Copyright (c) 2015 Tim Rozet (Red Hat), Dan Radez (Red Hat) and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10 set -e
  11 declare -i CNT
  12
  13 #rdo_images_uri=https://repos.fedorapeople.org/repos/openstack-m/rdo-images-centos-liberty-opnfv
  14 rdo_images_uri=file:///stable-images
  15 onos_artifacts_uri=file:///stable-images/onos
  16
  17 vm_index=4
  18 RDO_RELEASE=liberty
  19 SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null)
  20 OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network"
  21
  22 # check for dependancy packages
  23 for i in rpm-build createrepo libguestfs-tools python-docutils bsdtar; do
  24     if ! rpm -q $i > /dev/null; then
  25         sudo yum install -y $i
  26     fi
  27 done
  28
  29 # RDO Manager expects a stack user to exist, this checks for one
  30 # and creates it if you are root
  31 if ! id stack > /dev/null; then
  32     sudo useradd stack;
  33     sudo echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack
  34     sudo echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack
  35     sudo chmod 0440 /etc/sudoers.d/stack
  36     echo 'Added user stack'
  37 fi
  38
  39 # ensure that I can ssh as the stack user
  40 if ! sudo grep "$(cat ~/.ssh/id_rsa.pub)" /home/stack/.ssh/authorized_keys; then
  41     if ! sudo ls -d /home/stack/.ssh/ ; then
  42         sudo mkdir /home/stack/.ssh
  43         sudo chown stack:stack /home/stack/.ssh
  44         sudo chmod 700 /home/stack/.ssh
  45     fi
  46     USER=$(whoami) sudo sh -c "cat ~$USER/.ssh/id_rsa.pub >> /home/stack/.ssh/authorized_keys"
  47     sudo chown stack:stack /home/stack/.ssh/authorized_keys
  48 fi
  49
  50 # clean up stack user previously build instack disk images
  51 ssh -T ${SSH_OPTIONS[@]} stack@localhost "rm -f instack*.qcow2"
  52
  53 # Yum repo setup for building the undercloud
  54 if ! rpm -q rdo-release > /dev/null && [ "$1" != "-master" ]; then
  55     #pulling from current-passed-ci instead of release repos
  56     #sudo yum install -y https://rdoproject.org/repos/openstack-${RDO_RELEASE}/rdo-release-${RDO_RELEASE}.rpm
  57     sudo yum -y install yum-plugin-priorities
  58     sudo yum-config-manager --disable openstack-${RDO_RELEASE}
  59     sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo
  60     sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
  61     sudo rm -f /etc/yum.repos.d/delorean-current.repo
  62 elif [ "$1" == "-master" ]; then
  63     sudo yum -y install yum-plugin-priorities
  64     sudo yum-config-manager --disable openstack-${RDO_RELEASE}
  65     sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo
  66     sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
  67     sudo rm -f /etc/yum.repos.d/delorean-current.repo
  68 fi
  69
  70 # install the opendaylight yum repo definition
  71 cat << 'EOF' | sudo tee /etc/yum.repos.d/opendaylight.repo
  72 [opendaylight]
  73 name=OpenDaylight $releasever - $basearch
  74 baseurl=http://cbs.centos.org/repos/nfv7-opendaylight-33-release/$basearch/os/
  75 enabled=1
  76 gpgcheck=0
  77 EOF
  78
  79 # ensure the undercloud package is installed so we can build the undercloud
  80 if ! rpm -q instack-undercloud > /dev/null; then
  81     sudo yum install -y python-tripleoclient
  82 fi
  83
  84 # ensure openvswitch is installed
  85 if ! rpm -q openvswitch > /dev/null; then
  86     sudo yum install -y openvswitch
  87 fi
  88
  89 # ensure libvirt is installed
  90 if ! rpm -q libvirt-daemon-kvm > /dev/null; then
  91     sudo yum install -y libvirt-daemon-kvm
  92 fi
  93
  94 # clean this up incase it's there
  95 sudo rm -f /tmp/instack.answers
  96
  97 # ensure that no previous undercloud VMs are running
  98 sudo ../ci/clean.sh
  99 # and rebuild the bare undercloud VMs
 100 ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
 101 set -e
 102 NODE_COUNT=5 NODE_CPU=2 NODE_MEM=8192 TESTENV_ARGS="--baremetal-bridge-names 'brbm brbm1 brbm2 brbm3'" instack-virt-setup
 103 EOI
 104
 105 # let dhcp happen so we can get the ip
 106 # just wait instead of checking until we see an address
 107 # because there may be a previous lease that needs
 108 # to be cleaned up
 109 sleep 5
 110
 111 # get the undercloud ip address
 112 UNDERCLOUD=$(grep instack /var/lib/libvirt/dnsmasq/default.leases | awk '{print $3}' | head -n 1)
 113 if [ -z "$UNDERCLOUD" ]; then
 114   #if not found then dnsmasq may be using leasefile-ro
 115   instack_mac=$(ssh -T ${SSH_OPTIONS[@]} stack@localhost "virsh domiflist instack" | grep default | \
 116                 grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+")
 117   UNDERCLOUD=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'})
 118
 119   if [ -z "$UNDERCLOUD" ]; then
 120     echo "\n\nNever got IP for Instack. Can Not Continue."
 121     exit 1
 122   else
 123     echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
 124   fi
 125 else
 126    echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
 127 fi
 128
 129 # ensure that we can ssh to the undercloud
 130 CNT=10
 131 while ! ssh -T ${SSH_OPTIONS[@]}  "root@$UNDERCLOUD" "echo ''" > /dev/null && [ $CNT -gt 0 ]; do
 132     echo -n "."
 133     sleep 3
 134     CNT=CNT-1
 135 done
 136 # TODO fail if CNT=0
 137
 138 # yum repo, triple-o package and ssh key setup for the undercloud
 139 ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
 140 set -e
 141
 142 if ! rpm -q epel-release > /dev/null; then
 143     yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 144 fi
 145
 146 yum -y install yum-plugin-priorities
 147 curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo
 148 curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
 149
 150 cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys
 151 chown stack:stack /home/stack/.ssh/authorized_keys
 152 EOI
 153
 154 # copy instackenv file for future virt deployments
 155 if [ ! -d stack ]; then mkdir stack; fi
 156 scp ${SSH_OPTIONS[@]} stack@$UNDERCLOUD:instackenv.json stack/instackenv.json
 157
 158 # make a copy of instack VM's definitions, and disk image
 159 # it must be stopped to make a copy of its disk image
 160 ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
 161 set -e
 162 echo "Shutting down instack to gather configs"
 163 virsh shutdown instack
 164
 165 echo "Waiting for instack VM to shutdown"
 166 CNT=20
 167 while virsh list | grep instack > /dev/null && [ $CNT -gt 0 ]; do
 168     echo -n "."
 169     sleep 5
 170     CNT=CNT-1
 171 done
 172 if virsh list | grep instack > /dev/null; then
 173     echo "instack failed to shutdown for copy"
 174     exit 1
 175 fi
 176
 177 echo $'\nGenerating libvirt configuration'
 178 for i in \$(seq 0 $vm_index); do
 179   virsh dumpxml baremetalbrbm_brbm1_brbm2_brbm3_\$i | awk '/model type='\''virtio'\''/{c++;if(c==2){sub("model type='\''virtio'\''","model type='\''rtl8139'\''");c=0}}1' > baremetalbrbm_brbm1_brbm2_brbm3_\$i.xml
 180 done
 181
 182 virsh dumpxml instack > instack.xml
 183 virsh net-dumpxml brbm > brbm-net.xml
 184 virsh net-dumpxml brbm1 > brbm1-net.xml
 185 virsh net-dumpxml brbm2> brbm2-net.xml
 186 virsh net-dumpxml brbm3 > brbm3-net.xml
 187 virsh pool-dumpxml default > default-pool.xml
 188 EOI
 189
 190 # copy off the instack artifacts
 191 echo "Copying instack files to build directory"
 192 for i in $(seq 0 $vm_index); do
 193   scp ${SSH_OPTIONS[@]} stack@localhost:baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml .
 194 done
 195
 196 scp ${SSH_OPTIONS[@]} stack@localhost:instack.xml .
 197 scp ${SSH_OPTIONS[@]} stack@localhost:brbm-net.xml .
 198 scp ${SSH_OPTIONS[@]} stack@localhost:brbm1-net.xml .
 199 scp ${SSH_OPTIONS[@]} stack@localhost:brbm2-net.xml .
 200 scp ${SSH_OPTIONS[@]} stack@localhost:brbm3-net.xml .
 201 scp ${SSH_OPTIONS[@]} stack@localhost:default-pool.xml .
 202
 203 # pull down the the built images
 204 echo "Copying overcloud resources"
 205 IMAGES="overcloud-full.tar"
 206 IMAGES+=" undercloud.qcow2"
 207
 208 for i in $IMAGES; do
 209   # download prebuilt images from RDO Project
 210   if [ "$(curl -L $rdo_images_uri/${i}.md5 | awk {'print $1'})" != "$(md5sum stack/$i | awk {'print $1'})" ] ; then
 211     #if [ $i == "undercloud.qcow2" ]; then
 212     ### there's a problem with the Content-Length reported by the centos artifacts
 213     ### server so using wget for it until a resolution is figured out.
 214     #wget -nv -O stack/$i $rdo_images_uri/$i
 215     #else
 216     curl $rdo_images_uri/$i -o stack/$i
 217     #fi
 218   fi
 219   # only untar the tar files
 220   if [ "${i##*.}" == "tar" ]; then tar -xf stack/$i -C stack/; fi
 221 done
 222
 223 #Adding OpenStack packages to undercloud
 224 pushd stack
 225 cp undercloud.qcow2 instack.qcow2
 226 LIBGUESTFS_BACKEND=direct virt-customize --install yum-priorities -a instack.qcow2
 227 PACKAGES="qemu-kvm-common,qemu-kvm,libvirt-daemon-kvm,libguestfs,python-libguestfs,openstack-nova-compute"
 228 PACKAGES+=",openstack-swift,openstack-ceilometer-api,openstack-neutron-ml2,openstack-ceilometer-alarm"
 229 PACKAGES+=",openstack-nova-conductor,openstack-ironic-inspector,openstack-ironic-api,python-openvswitch"
 230 PACKAGES+=",openstack-glance,python-glance,python-troveclient,openstack-puppet-modules"
 231 PACKAGES+=",openstack-neutron,openstack-neutron-openvswitch,openstack-nova-scheduler,openstack-keystone,openstack-swift-account"
 232 PACKAGES+=",openstack-swift-container,openstack-swift-object,openstack-swift-plugin-swift3,openstack-swift-proxy"
 233 PACKAGES+=",openstack-nova-api,openstack-nova-cert,openstack-heat-api-cfn,openstack-heat-api,"
 234 PACKAGES+=",openstack-ceilometer-central,openstack-ceilometer-polling,openstack-ceilometer-collector,"
 235 PACKAGES+=",openstack-heat-api-cloudwatch,openstack-heat-engine,openstack-heat-common,openstack-ceilometer-notification"
 236 PACKAGES+=",hiera,puppet,memcached,keepalived,mariadb,mariadb-server,rabbitmq-server,python-pbr,python-proliantutils"
 237 PACKAGES+=",ceph-common"
 238
 239 # install the packages above and enabling ceph to live on the controller
 240 LIBGUESTFS_BACKEND=direct virt-customize --install $PACKAGES \
 241     --run-command "sed -i '/ControllerEnableCephStorage/c\\  ControllerEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \
 242     --run-command "sed -i '/  \$enable_ceph = /c\\  \$enable_ceph = true' /usr/share/openstack-tripleo-heat-templates/puppet/manifests/overcloud_controller_pacemaker.pp" \
 243     --run-command "sed -i '/  \$enable_ceph = /c\\  \$enable_ceph = true' /usr/share/openstack-tripleo-heat-templates/puppet/manifests/overcloud_controller.pp" \
 244     -a instack.qcow2
 245 popd
 246
 247
 248 pushd stack
 249
 250 ##########################################################
 251 #####  Prep initial overcloud image with common deps #####
 252 ##########################################################
 253
 254 # make a copy of the cached overcloud-full image
 255 cp overcloud-full.qcow2 overcloud-full-opendaylight.qcow2
 256 # Update puppet-aodh it's old
 257 rm -rf aodh
 258 git clone https://github.com/openstack/puppet-aodh aodh
 259 pushd aodh
 260 git checkout stable/liberty
 261 popd
 262 tar -czf puppet-aodh.tar.gz aodh
 263
 264 # Add epel, aodh and ceph, remove openstack-neutron-openvswitch
 265 AODH_PKG="openstack-aodh-api,openstack-aodh-common,openstack-aodh-compat,openstack-aodh-evaluator,openstack-aodh-expirer"
 266 AODH_PKG+=",openstack-aodh-listener,openstack-aodh-notifier"
 267 LIBGUESTFS_BACKEND=direct virt-customize --upload "/tmp/xfs-grow-remount-fix.service:/usr/lib/systemd/system/xfs-grow-remount-fix.service" \
 268     --run-command "systemctl enable xfs-grow-remount-fix.service" \
 269     --upload puppet-aodh.tar.gz:/etc/puppet/modules/ \
 270     --run-command "cd /etc/puppet/modules/ && rm -rf aodh && tar xzf puppet-aodh.tar.gz" \
 271     --run-command "yum remove -y openstack-neutron-openvswitch" \
 272     --run-command "echo 'nf_conntrack_proto_sctp' > /etc/modules-load.d/nf_conntrack_proto_sctp.conf" \
 273     --install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
 274     --install "$AODH_PKG,ceph" \
 275     -a overcloud-full-opendaylight.qcow2
 276
 277 ###############################################
 278 #####    Adding OpenDaylight to overcloud #####
 279 ###############################################
 280
 281 cat > /tmp/opendaylight.repo << EOF
 282 [opendaylight]
 283 name=OpenDaylight \$releasever - \$basearch
 284 baseurl=http://cbs.centos.org/repos/nfv7-opendaylight-33-release/\$basearch/os/
 285 enabled=1
 286 gpgcheck=0
 287 EOF
 288
 289 # install ODL packages
 290 LIBGUESTFS_BACKEND=direct virt-customize \
 291     --upload /tmp/opendaylight.repo:/etc/yum.repos.d/opendaylight.repo \
 292     --install opendaylight,python-networking-odl \
 293     -a overcloud-full-opendaylight.qcow2
 294
 295 ## WORK AROUND
 296 ## when OpenDaylight lands in upstream RDO manager this can be removed
 297
 298 # upload the opendaylight puppet module
 299 rm -rf puppet-opendaylight
 300 git clone -b 2.2.0 https://github.com/dfarrell07/puppet-opendaylight
 301 pushd puppet-opendaylight
 302 git archive --format=tar.gz --prefix=opendaylight/ HEAD > ../puppet-opendaylight.tar.gz
 303 popd
 304 LIBGUESTFS_BACKEND=direct virt-customize --upload puppet-opendaylight.tar.gz:/etc/puppet/modules/ \
 305                                          --run-command "cd /etc/puppet/modules/ && tar xzf puppet-opendaylight.tar.gz" \
 306                                          --upload ../opendaylight-puppet-neutron.patch:/tmp \
 307                                          --run-command "cd /etc/puppet/modules/neutron && patch -Np1 < /tmp/opendaylight-puppet-neutron.patch" \
 308                                          -a overcloud-full-opendaylight.qcow2
 309
 310 # Patch in OpenDaylight installation and configuration
 311 LIBGUESTFS_BACKEND=direct virt-customize --upload ../opnfv-tripleo-heat-templates.patch:/tmp \
 312                                          --run-command "cd /usr/share/openstack-tripleo-heat-templates/ && patch -Np1 < /tmp/opnfv-tripleo-heat-templates.patch" \
 313                                          -a instack.qcow2
 314
 315 # REMOVE ME AFTER Brahmaputra
 316 LIBGUESTFS_BACKEND=direct virt-customize --upload ../puppet-neutron-force-metadata.patch:/tmp \
 317                                          --run-command "cd /etc/puppet/modules/neutron && patch -Np1 < /tmp/puppet-neutron-force-metadata.patch" \
 318                                          -a overcloud-full-opendaylight.qcow2
 319
 320 LIBGUESTFS_BACKEND=direct virt-customize --upload ../puppet-cinder-quota-fix.patch:/tmp \
 321                                          --run-command "cd /etc/puppet/modules/cinder && patch -Np1 < /tmp/puppet-cinder-quota-fix.patch" \
 322                                          -a overcloud-full-opendaylight.qcow2
 323
 324 LIBGUESTFS_BACKEND=direct virt-customize --upload ../aodh-puppet-tripleo.patch:/tmp \
 325                                          --run-command "cd /etc/puppet/modules/tripleo && patch -Np1 < /tmp/aodh-puppet-tripleo.patch" \
 326                                          -a overcloud-full-opendaylight.qcow2
 327
 328 # adds tripleoclient aodh workaround
 329 # for keystone
 330 LIBGUESTFS_BACKEND=direct virt-customize --upload ../aodh-tripleoclient.patch:/tmp \
 331                                          --run-command "cd /usr/lib/python2.7/site-packages/tripleoclient && patch -Np1 < /tmp/aodh-tripleoclient.patch" \
 332                                          --upload ../aodh-os-cloud-config.patch:/tmp \
 333                                          --run-command "cd /usr/lib/python2.7/site-packages/os_cloud_config && patch -Np1 < /tmp/aodh-os-cloud-config.patch" \
 334                                          -a instack.qcow2
 335 # END REMOVE ME AFTER Brahmaputra
 336
 337 ################################################
 338 #####    Adding SFC+OpenDaylight overcloud #####
 339 ################################################
 340
 341 cat > /tmp/opendaylight.repo << EOF
 342 [opendaylight]
 343 name=OpenDaylight \$releasever - \$basearch
 344 baseurl=http://cbs.centos.org/repos/nfv7-opendaylight-4-testing/\$basearch/os/
 345 enabled=1
 346 gpgcheck=0
 347 EOF
 348
 349 #copy opendaylight overcloud full to isolate odl-sfc
 350 cp overcloud-full-opendaylight.qcow2 overcloud-full-opendaylight-sfc.qcow2
 351
 352 # upload the opendaylight puppet module
 353 rm -rf puppet-opendaylight
 354 git clone -b 3.0.1 https://github.com/dfarrell07/puppet-opendaylight
 355 pushd puppet-opendaylight
 356 git archive --format=tar.gz --prefix=opendaylight/ HEAD > ../puppet-opendaylight.tar.gz
 357 popd
 358
 359 # kernel is patched with patch from this post
 360 # http://xfs.org/index.php/XFS_FAQ#Q:_Why_do_I_receive_No_space_left_on_device_after_xfs_growfs.3F
 361 LIBGUESTFS_BACKEND=direct virt-customize \
 362     --install 'https://radez.fedorapeople.org/kernel-ml-3.13.7-1.el7.centos_xfs_grow.x86_64.rpm' \
 363     --run-command 'grub2-set-default "\$(grep -P \"submenu|^menuentry\" /boot/grub2/grub.cfg | cut -d \"\\x27\" | head -n 1)"' \
 364     --install 'https://radez.fedorapeople.org/openvswitch-kmod-2.3.90-1.el7.centos.x86_64.rpm' \
 365     --run-command 'yum downgrade -y https://radez.fedorapeople.org/openvswitch-2.3.90-1.x86_64.rpm' \
 366     --run-command 'rm -f /lib/modules/3.13.7-1.el7.centos_xfs_grow.x86_64/kernel/net/openvswitch/openvswitch.ko' \
 367     --run-command 'ln -s /lib/modules/3.13.7-1.el7.centos_xfs_grow.x86_64/kernel/extra/openvswitch/openvswitch.ko /lib/modules/3.13.7-1.el7.centos_xfs_grow.x86_64/kernel/net/openvswitch/openvswitch.ko' \
 368     --upload /tmp/opendaylight.repo:/etc/yum.repos.d/opendaylight.repo \
 369     --run-command "yum remove -y opendaylight" \
 370     --run-command "yum clean all" \
 371     --run-command "yum install -y opendaylight" \
 372     --run-command "rm -rf /etc/puppet/modules/opendaylight && rm -f /etc/puppet/modules/puppet-opendaylight.tar.gz " \
 373     --upload puppet-opendaylight.tar.gz:/etc/puppet/modules/ \
 374     --run-command "cd /etc/puppet/modules/ && tar xzf puppet-opendaylight.tar.gz" \
 375     -a overcloud-full-opendaylight-sfc.qcow2
 376
 377
 378
 379 ###############################################
 380 #####    Adding ONOS to overcloud #####
 381 ###############################################
 382
 383 ## WORK AROUND
 384 ## when ONOS lands in upstream OPNFV artifacts this can be removed
 385
 386 # upload the onos puppet module
 387
 388 rm -rf puppet-onos
 389 git clone https://github.com/bobzhouHW/puppet-onos.git
 390 pushd puppet-onos
 391 # download jdk, onos and maven dependancy packages.
 392 pushd files
 393 curl ${onos_artifacts_uri}/jdk-8u51-linux-x64.tar.gz -o ./jdk-8u51-linux-x64.tar.gz
 394 curl ${onos_artifacts_uri}/onos-1.3.0.tar.gz -o ./onos-1.3.0.tar.gz
 395 curl ${onos_artifacts_uri}/repository.tar -o ./repository.tar
 396 popd
 397 popd
 398 mv puppet-onos onos
 399 tar -czf puppet-onos.tar.gz onos
 400 LIBGUESTFS_BACKEND=direct virt-customize --upload puppet-onos.tar.gz:/etc/puppet/modules/ \
 401                                          --run-command "cd /etc/puppet/modules/ && tar xzf puppet-onos.tar.gz" -a overcloud-full-opendaylight.qcow2
 402
 403 ## END WORK AROUND
 404
 405 popd
 406
 407 # move and Sanitize private keys from instack.json file
 408 mv stack/instackenv.json instackenv-virt.json
 409 sed -i '/pm_password/c\      "pm_password": "INSERT_STACK_USER_PRIV_KEY",' instackenv-virt.json
 410 sed -i '/ssh-key/c\  "ssh-key": "INSERT_STACK_USER_PRIV_KEY",' instackenv-virt.json
 411
 412 # clean up the VMs
 413 ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
 414 set -e
 415 virsh destroy instack 2> /dev/null || echo -n ''
 416 virsh undefine instack --remove-all-storage 2> /dev/null || echo -n ''
 417 for i in \$(seq 0 $vm_index); do
 418   virsh destroy baremetalbrbm_brbm1_brbm2_brbm3_\$i 2> /dev/null || echo -n ''
 419   virsh undefine baremetalbrbm_brbm1_brbm2_brbm3_\$i --remove-all-storage 2> /dev/null || echo -n ''
 420 done
 421 EOI
 422