# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+# Configuration
+
[[ "${CI_DEBUG:-true}" == [Tt]rue ]] && set -x
IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
DOCTOR_PROJECT=doctor
#TODO: change back to `_member_` when JIRA DOCTOR-55 is done
DOCTOR_ROLE=admin
+PROFILER_TYPE=${PROFILER_TYPE:-none}
-SUPPORTED_INSPECTOR_TYPES="sample congress"
-INSPECTOR_TYPE=${INSPECTOR_TYPE:-sample}
TOP_DIR=$(cd $(dirname "$0") && pwd)
ssh_opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW
--os-tenant-name $DOCTOR_PROJECT"
-if [[ ! "$SUPPORTED_INSPECTOR_TYPES" =~ "$INSPECTOR_TYPE" ]] ; then
- echo "ERROR: INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
- exit 1
-fi
+
+# Functions
get_compute_host_info() {
# get computer host info which VM boot in
sudo kill $(pgrep -f "python monitor.py")
}
-congress_add_rule() {
- name=$1
- policy=$2
- rule=$3
-
- if ! openstack congress policy rule list $policy | grep -q -e "// Name: $name$" ; then
- openstack congress policy rule create --name $name $policy "$rule"
- fi
-}
-
-congress_del_rule() {
- name=$1
- policy=$2
-
- if openstack congress policy rule list $policy | grep -q -e "^// Name: $name$" ; then
- openstack congress policy rule delete $policy $name
- fi
-}
-
-congress_setup_rules() {
- congress_add_rule host_down classification \
- 'host_down(host) :-
- doctor:events(hostname=host, type="compute.host.down", status="down")'
-
- congress_add_rule active_instance_in_host classification \
- 'active_instance_in_host(vmid, host) :-
- nova:servers(id=vmid, host_name=host, status="ACTIVE")'
-
- congress_add_rule host_force_down classification \
- 'execute[nova:services.force_down(host, "nova-compute", "True")] :-
- host_down(host)'
-
- congress_add_rule error_vm_states classification \
- 'execute[nova:servers.reset_state(vmid, "error")] :-
- host_down(host),
- active_instance_in_host(vmid, host)'
-}
-
-start_inspector() {
- if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then
- pgrep -f "python inspector.py" && return 0
- python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
- elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then
- nova_api_min_version="2.11"
- nova_api_version=$(openstack congress datasource list | \
- grep nova | grep -Po "(?<='api_version': ')[^']*")
- if ! is_set nova_api_version; then
- nova_api_version="2.0"
- fi
- if [[ "$nova_api_version" < "$nova_api_min_version" ]]; then
- die $LINENO "Congress Nova datasource API version < $nova_api_min_version ($nova_api_version)"
- fi
- openstack congress driver list | grep -q " doctor "
- openstack congress datasource list | grep -q " doctor " || {
- openstack congress datasource create doctor doctor
- }
- congress_setup_rules
- fi
-}
-
-stop_inspector() {
- if [[ "$INSPECTOR_TYPE" == "sample" ]] ; then
- pgrep -f "python inspector.py" || return 0
- kill $(pgrep -f "python inspector.py")
- elif [[ "$INSPECTOR_TYPE" == "congress" ]] ; then
- congress_del_rule host_force_down classification
- congress_del_rule error_vm_states classification
- congress_del_rule active_instance_in_host classification
- congress_del_rule host_down classification
- fi
-}
-
start_consumer() {
pgrep -f "python consumer.py" && return 0
python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 &
dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $7}')
sleep 1
sudo ip link set $dev down
+echo "doctor set host down at" $(date "+%s.%N")
sleep 180
sudo ip link set $dev up
sleep 1
ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
}
+profile_performance_poc() {
+ triggered=$(grep "^doctor set host down at" disable_network.log |\
+ sed -e "s/^.* at //")
+ vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\
+ sed -e "s/^.* at //")
+ hostdown=$(grep "doctor mark host.* down at" inspector.log |\
+ sed -e "s/^.* at //")
+
+ #calculate the relative interval to triggered(T00)
+ export DOCTOR_PROFILER_T00=0
+ export DOCTOR_PROFILER_T01=$(echo "($detected-$triggered)*1000/1" |bc)
+ export DOCTOR_PROFILER_T03=$(echo "($vmdown-$triggered)*1000/1" |bc)
+ export DOCTOR_PROFILER_T04=$(echo "($hostdown-$triggered)*1000/1" |bc)
+ export DOCTOR_PROFILER_T09=$(echo "($notified-$triggered)*1000/1" |bc)
+
+ python profiler-poc.py
+}
+
calculate_notification_time() {
- detected=$(grep "doctor monitor detected at" monitor.log | awk '{print $10}')
- notified=$(grep "doctor consumer notified at" consumer.log | awk '{print $10}')
if ! grep -q "doctor consumer notified at" consumer.log ; then
die $LINENO "Consumer hasn't received fault notification."
fi
+
+ #keep 'at' as the last keyword just before the value, and
+ #use regex to get value instead of the fixed column
+ detected=$(grep "doctor monitor detected at" monitor.log |\
+ sed -e "s/^.* at //")
+ notified=$(grep "doctor consumer notified at" consumer.log |\
+ sed -e "s/^.* at //")
+
+ if [[ "$PROFILER_TYPE" == "poc" ]]; then
+ profile_performance_poc
+ fi
+
echo "$notified $detected" | \
awk '{
d = $1 - $2;
openstack user delete "$DOCTOR_USER"
cleanup_installer
+ cleanup_inspector
}
+# Main process
echo "Note: doctor/tests/run.sh has been executed."
source $TOP_DIR/functions-common
source $TOP_DIR/lib/installer
+source $TOP_DIR/lib/inspector
setup_installer
Code Review / doctor.git / blobdiff