Ensure cluster remains stable during services restarts

Using crm_resource --wait we wait for the cluster to get into
a stable state before moving into the next step of the piloted
restart procedure.

Change-Id: I80199653024383fd07900dad0b8d23fb8afade26
Co-Authored-By: Jiri Stransky <jistr@redhat.com>

diff --git a/extraconfig/tasks/pacemaker_resource_restart.sh b/extraconfig/tasks/pacemaker_resource_restart.sh
index 4e8b20f..1220109 100755 (executable)
--- a/extraconfig/tasks/pacemaker_resource_restart.sh
+++ b/extraconfig/tasks/pacemaker_resource_restart.sh
@@ -26,6 +26,7 @@ function check_resource {
           sleep $check_interval
       else
         echo "$service has $state"
+        timeout -k 10 $timeout crm_resource --wait
         return
       fi
   done
@@ -50,7 +51,7 @@ if [ "$pacemaker_status" = "active" -a \
     pcs resource disable httpd
     check_resource httpd stopped 300
     pcs resource disable openstack-keystone
-    check_resource openstack-keystone stopped 1200
+    check_resource openstack-keystone stopped 1800
 
     if pcs status | grep haproxy-clone; then
         pcs resource restart haproxy-clone
@@ -62,7 +63,7 @@ if [ "$pacemaker_status" = "active" -a \
     pcs resource restart galera-master
 
     pcs resource enable openstack-keystone
-    check_resource openstack-keystone started 300
+    check_resource openstack-keystone started 1800
     pcs resource enable httpd
     check_resource httpd started 800