Verify galera is sync'd in yum_update.sh

When the cluster is brought back online after a yum update in
yum_update.sh, we should verify that galera is fully sync'd before
moving on. This ensures the sync is complete before moving on to update
any other nodes in the cluster.

Change-Id: Ie8fc2c5d5214deacea94ca658ac75359b318ced1

diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh
index 3ba13f2..fa523e8 100755 (executable)
--- a/extraconfig/tasks/yum_update.sh
+++ b/extraconfig/tasks/yum_update.sh
@@ -23,6 +23,7 @@ update_identifier=${update_identifier//[^a-zA-Z0-9-_]/}
 
 # seconds to wait for this node to rejoin the cluster after update
 cluster_start_timeout=360
+galera_sync_timeout=360
 
 timestamp_file="$timestamp_dir/$update_identifier"
 if [[ -a "$timestamp_file" ]]; then
@@ -185,6 +186,17 @@ if [[ "$pacemaker_status" == "active" ]] ; then
             exit 1
         fi
     done
+
+    tstart=$(date +%s)
+    while ! clustercheck; do
+        sleep 5
+        tnow=$(date +%s)
+        if (( tnow-tstart > galera_sync_timeout )) ; then
+            echo "ERROR galera sync timed out"
+            exit 1
+        fi
+    done
+
     pcs status
 
 else