+ wait_consumer 60
+ #keep 'at' as the last keyword just before the value, and
+ #use regex to get value instead of the fixed column
+ if [ ! -f monitor.log ]; then
+ scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" .
+ fi
+ detected=$(grep "doctor monitor detected at" monitor.log |\
+ sed -e "s/^.* at //" | tail -1)
+ notified=$(grep "doctor consumer notified at" consumer.log |\
+ sed -e "s/^.* at //" | tail -1)
+
+ echo "$notified $detected" | \
+ awk '{
+ d = $1 - $2;
+ if (d < 1 && d > 0) { print d " OK"; exit 0 }
+ else { print d " NG"; exit 1 }
+ }'
+}
+
+check_host_status() {
+ # Check host related to first Doctor VM is in wanted state
+ # $1 Expected state
+ # $2 Seconds to wait to have wanted state
+ expected_state=$1
+ local interval=5
+ local rounds=$(($2 / $interval))
+ for i in `seq $rounds`; do
+ host_status_line=$(openstack $as_doctor_user --os-compute-api-version \
+ 2.16 server show ${VM_BASENAME}1 | grep "host_status")
+ host_status=$(echo $host_status_line | awk '{print $4}')
+ die_if_not_set $LINENO host_status "host_status not reported by: nova show ${VM_BASENAME}1"
+ if [[ "$expected_state" =~ "$host_status" ]] ; then
+ echo "${VM_BASENAME}1 showing host_status: $host_status"
+ return 0
+ else
+ sleep $interval
+ fi
+ done
+ if [[ "$expected_state" =~ "$host_status" ]] ; then
+ echo "${VM_BASENAME}1 showing host_status: $host_status"
+ else
+ die $LINENO "host_status:$host_status not equal to expected_state: $expected_state"
+ fi
+}
+
+unset_forced_down_hosts() {
+ # for debug
+ openstack compute service list --service nova-compute
+
+ downed_computes=$(openstack compute service list --service nova-compute \
+ -f value -c Host -c State | grep ' down$' \
+ | sed -e 's/ *down$//')
+ echo "downed_computes: $downed_computes"
+ for host in $downed_computes
+ do
+ # TODO(r-mibu): use openstack client
+ #openstack compute service set --up $host nova-compute
+ nova service-force-down --unset $host nova-compute
+ done
+
+ echo "waiting disabled compute host back to be enabled..."
+ wait_until 'openstack compute service list --service nova-compute
+ -f value -c State | grep -q down' 240 5
+
+ for host in $downed_computes
+ do
+ # TODO(r-mibu): improve 'get_compute_ip_from_hostname'
+ get_compute_ip_from_hostname $host
+ wait_until "! ping -c 1 $COMPUTE_IP" 120 5
+ done
+}
+
+collect_logs() {
+ if [[ -n "$COMPUTE_IP" ]];then
+ scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
+ fi
+
+ # TODO(yujunz) collect other logs, e.g. nova, aodh