Disable VIPs before stopping cluster during version upgrade

If "pcs cluster stop --all" is executed on a controller that
happens to have a VIP on the internal network, pcs may use the
VIP as the source address for communication with another cluster
node.  When pacemaker is stopped this VIP goes away, and pcs never
receives a response from the other node.  This causes pcs to hang
indefinitely; eventually the upgrade times out and fails.

Disabling the VIPs before stopping the cluster avoids this
situation.

Change-Id: I6bc59120211af28456018640033ce3763c373bbb
Closes-Bug: 1577570

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
index 2cbdc53..f539922 100755 (executable)
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
@@ -24,6 +24,12 @@ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)
     check_resource memcached stopped 600
     pcs resource disable galera
     check_resource galera stopped 600
+    # Disable all VIPs before stopping the cluster, so that pcs doesn't use one as a source address:
+    #   https://bugzilla.redhat.com/show_bug.cgi?id=1330688
+    for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Started | awk '{ print $1 }'); do
+      pcs resource disable $vip
+      check_resource $vip stopped 60
+    done
     pcs cluster stop --all
 fi
 

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
index 28e279b..643ae57 100755 (executable)
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
@@ -24,6 +24,11 @@ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)
         exit 1
     fi
 
+    for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Stopped | awk '{ print $1 }'); do
+      pcs resource enable $vip
+      check_resource $vip started 60
+    done
+
     pcs resource enable galera
     check_resource galera started 600
     pcs resource enable mongod