+ die_if_not_set $LINENO host_status "host_status not reported by: nova show ${VM_BASENAME}1"
+ if [[ "$expected_state" =~ "$host_status" ]] ; then
+ echo "${VM_BASENAME}1 showing host_status: $host_status"
+ return 0
+ else
+ sleep $interval
+ fi
+ done
+ if [[ "$expected_state" =~ "$host_status" ]] ; then
+ echo "${VM_BASENAME}1 showing host_status: $host_status"
+ else
+ die $LINENO "host_status:$host_status not equal to expected_state: $expected_state"
+ fi
+}
+
+unset_forced_down_hosts() {
+ for host in $(openstack compute service list --service nova-compute \
+ -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
+ do
+ # TODO (r-mibu): make sample inspector use keystone v3 api
+ OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \
+ python ./nova_force_down.py $host --unset
+ done
+
+ echo "waiting disabled compute host back to be enabled..."
+ wait_until 'openstack compute service list --service nova-compute
+ -f value -c State | grep -q down' 240 5
+}
+
+collect_logs() {
+ unset_forced_down_hosts
+ # TODO: We need to make sure the target compute host is back to IP
+ # reachable. wait_ping() will be added by tojuvone .
+ sleep 110
+ scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
+
+ # TODO(yujunz) collect other logs, e.g. nova, aodh
+}
+
+run_profiler() {
+ if [[ "$PROFILER_TYPE" == "poc" ]]; then
+ linkdown=$(grep "doctor set link down at " disable_network.log |\
+ sed -e "s/^.* at //")
+ vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\
+ sed -e "s/^.* at //")
+ hostdown=$(grep "doctor mark host.* down at" inspector.log |\
+ sed -e "s/^.* at //")
+
+ # TODO(yujunz) check the actual delay to verify time sync status
+ # expected ~1s delay from $trigger to $linkdown
+ relative_start=${linkdown}
+ export DOCTOR_PROFILER_T00=$(python -c \
+ "print(int(($linkdown-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T01=$(python -c \
+ "print(int(($detected-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T03=$(python -c \
+ "print(int(($vmdown-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T04=$(python -c \
+ "print(int(($hostdown-$relative_start)*1000))")
+ export DOCTOR_PROFILER_T09=$(python -c \
+ "print(int(($notified-$relative_start)*1000))")
+
+ python profiler-poc.py >doctor_profiler.log 2>&1
+ fi