# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+# Configuration
+
[[ "${CI_DEBUG:-true}" == [Tt]rue ]] && set -x
IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
DOCTOR_PROJECT=doctor
#TODO: change back to `_member_` when JIRA DOCTOR-55 is done
DOCTOR_ROLE=admin
+PROFILER_TYPE=${PROFILER_TYPE:-none}
TOP_DIR=$(cd $(dirname "$0") && pwd)
--os-tenant-name $DOCTOR_PROJECT"
+# Functions
+
get_compute_host_info() {
# get computer host info which VM boot in
COMPUTE_HOST=$(openstack $as_doctor_user server show $VM_NAME |
return 0
fi
if [[ "$state" == "ERROR" ]]; then
+ openstack $as_doctor_user server show $VM_NAME
die $LINENO "vm state is ERROR"
fi
count=$(($count+1))
dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $7}')
sleep 1
sudo ip link set $dev down
+echo "doctor set host down at" $(date "+%s.%N")
sleep 180
sudo ip link set $dev up
sleep 1
ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
}
+profile_performance_poc() {
+ triggered=$(grep "^doctor set host down at" disable_network.log |\
+ sed -e "s/^.* at //")
+ vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\
+ sed -e "s/^.* at //")
+ hostdown=$(grep "doctor mark host.* down at" inspector.log |\
+ sed -e "s/^.* at //")
+
+ #calculate the relative interval to triggered(T00)
+ export DOCTOR_PROFILER_T00=0
+ export DOCTOR_PROFILER_T01=$(echo "($detected-$triggered)*1000/1" |bc)
+ export DOCTOR_PROFILER_T03=$(echo "($vmdown-$triggered)*1000/1" |bc)
+ export DOCTOR_PROFILER_T04=$(echo "($hostdown-$triggered)*1000/1" |bc)
+ export DOCTOR_PROFILER_T09=$(echo "($notified-$triggered)*1000/1" |bc)
+
+ python profiler-poc.py
+}
+
calculate_notification_time() {
- detected=$(grep "doctor monitor detected at" monitor.log | awk '{print $10}')
- notified=$(grep "doctor consumer notified at" consumer.log | awk '{print $10}')
if ! grep -q "doctor consumer notified at" consumer.log ; then
die $LINENO "Consumer hasn't received fault notification."
fi
+
+ #keep 'at' as the last keyword just before the value, and
+ #use regex to get value instead of the fixed column
+ detected=$(grep "doctor monitor detected at" monitor.log |\
+ sed -e "s/^.* at //")
+ notified=$(grep "doctor consumer notified at" consumer.log |\
+ sed -e "s/^.* at //")
+
+ if [[ "$PROFILER_TYPE" == "poc" ]]; then
+ profile_performance_poc
+ fi
+
echo "$notified $detected" | \
awk '{
d = $1 - $2;
fi
}
+unset_forced_down_hosts() {
+ for host in $(openstack compute service list --service nova-compute \
+ -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
+ do
+ python ./nova_force_down.py $host --unset
+ done
+
+ echo "waiting disabled compute host back to be enabled..."
+ wait_until 'openstack compute service list --service nova-compute
+ -f value -c State | grep -q down' 240 5
+}
+
cleanup() {
set +e
echo "cleanup..."
stop_inspector
stop_consumer
- echo "waiting disabled compute host back to be enabled..."
- python ./nova_force_down.py "$COMPUTE_HOST" --unset
- sleep 240
- check_host_status "UP"
+ unset_forced_down_hosts
+ # TODO: We need to make sure the target compute host is back to IP
+ # reachable. wait_ping() will be added by tojuvone .
+ sleep 110
scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
openstack $as_doctor_user server list | grep -q " $VM_NAME " && openstack $as_doctor_user server delete "$VM_NAME"
cleanup_inspector
}
+# Main process
echo "Note: doctor/tests/run.sh has been executed."
+git log --oneline -1 || true # ignore even you don't have git installed
trap cleanup EXIT