+check_host_status() {
+ # Check host related to first Doctor VM is in wanted state
+ # $1 Expected state
+ # $2 Seconds to wait to have wanted state
+ expected_state=$1
+ local interval=5
+ local rounds=$(($2 / $interval))
+ for i in `seq $rounds`; do
+ host_status_line=$(openstack $as_doctor_user --os-compute-api-version \
+ 2.16 server show ${VM_BASENAME}1 | grep "host_status")
+ host_status=$(echo $host_status_line | awk '{print $4}')
+ die_if_not_set $LINENO host_status "host_status not reported by: nova show ${VM_BASENAME}1"
+ if [[ "$expected_state" =~ "$host_status" ]] ; then
+ echo "${VM_BASENAME}1 showing host_status: $host_status"
+ return 0
+ else
+ sleep $interval
+ fi
+ done
+ if [[ "$expected_state" =~ "$host_status" ]] ; then
+ echo "${VM_BASENAME}1 showing host_status: $host_status"
+ else
+ die $LINENO "host_status:$host_status not equal to expected_state: $expected_state"
+ fi
+}
+
+unset_forced_down_hosts() {
+ # for debug
+ openstack compute service list --service nova-compute
+
+ downed_computes=$(openstack compute service list --service nova-compute \
+ -f value -c Host -c State | grep ' down$' \
+ | sed -e 's/ *down$//')
+ echo "downed_computes: $downed_computes"
+ for host in $downed_computes
+ do
+ # TODO(r-mibu): use openstack client
+ #openstack compute service set --up $host nova-compute
+ nova service-force-down --unset $host nova-compute
+ done
+
+ echo "waiting disabled compute host back to be enabled..."
+ wait_until 'openstack compute service list --service nova-compute
+ -f value -c State | grep -q down' 240 5
+
+ for host in $downed_computes
+ do
+ # TODO(r-mibu): improve 'get_compute_ip_from_hostname'
+ get_compute_ip_from_hostname $host
+ wait_until "! ping -c 1 $COMPUTE_IP" 120 5
+ done
+}
+
+collect_logs() {
+ if [[ -n "$COMPUTE_IP" ]];then
+ scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
+ fi
+
+ # TODO(yujunz) collect other logs, e.g. nova, aodh
+}
+
+run_profiler() {
+ if [[ "$PROFILER_TYPE" == "poc" ]]; then
+ linkdown=$(grep "doctor set link down at " disable_network.log |\
+ sed -e "s/^.* at //")
+ vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\
+ sed -e "s/^.* at //")
+ hostdown=$(grep "doctor mark host.* down at" inspector.log |\
+ sed -e "s/^.* at //")
+
+ # TODO(yujunz) check the actual delay to verify time sync status
+ # expected ~1s delay from $trigger to $linkdown
+ relative_start=${linkdown}
+ export DOCTOR_PROFILER_T00=$(python -c \
+ "print(int(($linkdown-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T01=$(python -c \
+ "print(int(($detected-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T03=$(python -c \
+ "print(int(($vmdown-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T04=$(python -c \
+ "print(int(($hostdown-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T09=$(python -c \
+ "print(int(($notified-$relative_start)*1000))")
+
+ python profiler-poc.py >doctor_profiler.log 2>&1
+ fi
+}
+
+cleanup() {
+ set +e
+ echo "cleanup..."
+ stop_monitor
+ stop_inspector
+ stop_consumer
+
+ unset_forced_down_hosts
+ collect_logs
+
+ vms=$(openstack $as_doctor_user server list)
+ vmstodel=""
+ for i in `seq $VM_COUNT`; do
+ $(echo "${vms}" | grep -q " $VM_BASENAME$i ") &&
+ vmstodel+=" $VM_BASENAME$i"
+ done
+ [[ $vmstodel ]] && openstack $as_doctor_user server delete $vmstodel
+ alarm_list=$($ceilometer alarm-list)
+ for i in `seq $VM_COUNT`; do
+ alarm_id=$(echo "${alarm_list}" | grep " $ALARM_BASENAME$i " |
+ awk '{print $2}')
+ [ -n "$alarm_id" ] && $ceilometer alarm-delete "$alarm_id"
+ done
+ openstack $as_doctor_user subnet delete $NET_NAME
+ sleep 1
+ openstack $as_doctor_user network delete $NET_NAME
+ sleep 1
+
+ image_id=$(openstack image list | grep " $IMAGE_NAME " | awk '{print $2}')
+ sleep 1
+ #if an existing image was used, there's no need to remove it here
+ if [[ "$use_existing_image" == false ]] ; then
+ [ -n "$image_id" ] && openstack image delete "$image_id"
+ fi
+ openstack role remove "$DOCTOR_ROLE" --user "$DOCTOR_USER" \
+ --project "$DOCTOR_PROJECT"
+ openstack project delete "$DOCTOR_PROJECT"
+ openstack user delete "$DOCTOR_USER"
+ # NOTE: remove role only for doctor test.
+ #openstack role delete "$DOCTOR_ROLE"
+
+ cleanup_installer
+ cleanup_inspector
+
+ # NOTE: Temporal log printer.
+ for f in $(find . -name '*.log')
+ do
+ echo
+ echo "[$f]"
+ sed -e 's/^/ | /' $f
+ echo
+ done
+}
+
+# Main process
+