$xtrace
}
+# Wait until the condition is met.
+# wait_until condition timeout interval
+function wait_until {
+ local condition="$1"
+ local timeout=$2
+ local interval=$3
+
+ while eval ${condition}
+ do
+ sleep ${interval}
+ timeout=$((${timeout} - ${interval}))
+ if [[ ${timeout} < 0 ]]; then
+ err $LINENO "timed out ($condition)..."
+ return 1
+ fi
+ done
+}
}
function start_inspector {
- if ! is_inspector_supported $INSPECTOR_TYPE; then
- die $LINENO"INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
- fi
-
- source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE
start_inspector_$INSPECTOR_TYPE
}
function cleanup_inspector {
cleanup_inspector_$INSPECTOR_TYPE
}
+
+
+if ! is_inspector_supported $INSPECTOR_TYPE; then
+ die $LINENO"INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
+fi
+
+source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE
}
function setup_installer {
- if ! is_installer_supported $INSTALLER_TYPE; then
- die $LINENO"INSTALLER_TYPE=$INSTALLER_TYPE is not supported."
- fi
-
- source $TOP_DIR/lib/installers/$INSTALLER_TYPE
-
if ! is_set INSTALLER_IP; then
get_installer_ip
fi
function cleanup_installer {
cleanup_installer_$INSTALLER_TYPE
}
+
+
+if ! is_installer_supported $INSTALLER_TYPE; then
+ die $LINENO"INSTALLER_TYPE=$INSTALLER_TYPE is not supported."
+fi
+
+source $TOP_DIR/lib/installers/$INSTALLER_TYPE
return 0
fi
if [[ "$state" == "ERROR" ]]; then
+ openstack $as_doctor_user server show $VM_NAME
die $LINENO "vm state is ERROR"
fi
count=$(($count+1))
fi
}
+unset_forced_down_hosts() {
+ for host in $(openstack compute service list --service nova-compute \
+ -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
+ do
+ python ./nova_force_down.py $host --unset
+ done
+
+ echo "waiting disabled compute host back to be enabled..."
+ wait_until 'openstack compute service list --service nova-compute
+ -f value -c State | grep -q down' 240 5
+}
+
cleanup() {
set +e
echo "cleanup..."
stop_inspector
stop_consumer
- echo "waiting disabled compute host back to be enabled..."
- python ./nova_force_down.py "$COMPUTE_HOST" --unset
- sleep 240
- check_host_status "UP"
+ unset_forced_down_hosts
+ # TODO: We need to make sure the target compute host is back to IP
+ # reachable. wait_ping() will be added by tojuvone .
+ sleep 110
scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
openstack $as_doctor_user server list | grep -q " $VM_NAME " && openstack $as_doctor_user server delete "$VM_NAME"