Fix functions for fuel installer

[doctor.git] / tests / run.sh

diff --git a/tests/run.sh b/tests/run.sh
index 1f73f55..227c6ea 100755 (executable)
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -27,6 +27,7 @@ DOCTOR_PW=doctor
 DOCTOR_PROJECT=doctor
 #TODO: change back to `_member_` when JIRA DOCTOR-55 is done
 DOCTOR_ROLE=admin
+PROFILER_TYPE=${PROFILER_TYPE:-none}
 
 TOP_DIR=$(cd $(dirname "$0") && pwd)
 
@@ -178,8 +179,7 @@ start_consumer() {
                              nova list | grep ' overcloud-controller-[0-9] ' \
                              | sed -e 's/^.*ctlplane=//' -e 's/ *|\$//'")
         elif is_installer fuel; then
-            CONTROLLER_IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \
-                            "fuel node | grep controller | cut -d '|' -f 5|xargs")
+            get_controller_ips
         fi
 
         die_if_not_set $LINENO CONTROLLER_IPS "Could not get CONTROLLER_IPS."
@@ -221,6 +221,7 @@ wait_for_vm_launch() {
             return 0
         fi
         if [[ "$state" == "ERROR" ]]; then
+            openstack $as_doctor_user server show $VM_NAME
             die $LINENO "vm state is ERROR"
         fi
         count=$(($count+1))
@@ -234,8 +235,10 @@ inject_failure() {
     cat > disable_network.sh << 'END_TXT'
 #!/bin/bash -x
 dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $7}')
+[[ -n "$dev" ]] || dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $5}')
 sleep 1
 sudo ip link set $dev down
+echo "doctor set host down at" $(date "+%s.%N")
 sleep 180
 sudo ip link set $dev up
 sleep 1
@@ -246,12 +249,40 @@ END_TXT
     ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
 }
 
+profile_performance_poc() {
+    triggered=$(grep "^doctor set host down at" disable_network.log |\
+                sed -e "s/^.* at //")
+    vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\
+               sed -e "s/^.* at //")
+    hostdown=$(grep "doctor mark host.* down at" inspector.log |\
+               sed -e "s/^.* at //")
+
+    #calculate the relative interval to triggered(T00)
+    export DOCTOR_PROFILER_T00=0
+    export DOCTOR_PROFILER_T01=$(echo "($detected-$triggered)*1000/1" |bc)
+    export DOCTOR_PROFILER_T03=$(echo "($vmdown-$triggered)*1000/1" |bc)
+    export DOCTOR_PROFILER_T04=$(echo "($hostdown-$triggered)*1000/1" |bc)
+    export DOCTOR_PROFILER_T09=$(echo "($notified-$triggered)*1000/1" |bc)
+
+    python profiler-poc.py
+}
+
 calculate_notification_time() {
-    detected=$(grep "doctor monitor detected at" monitor.log | awk '{print $10}')
-    notified=$(grep "doctor consumer notified at" consumer.log | awk '{print $10}')
     if ! grep -q "doctor consumer notified at" consumer.log ; then
         die $LINENO "Consumer hasn't received fault notification."
     fi
+
+    #keep 'at' as the last keyword just before the value, and
+    #use regex to get value instead of the fixed column
+    detected=$(grep "doctor monitor detected at" monitor.log |\
+               sed -e "s/^.* at //")
+    notified=$(grep "doctor consumer notified at" consumer.log |\
+               sed -e "s/^.* at //")
+
+    if [[ "$PROFILER_TYPE" == "poc" ]]; then
+        profile_performance_poc
+    fi
+
     echo "$notified $detected" | \
         awk '{
             d = $1 - $2;
@@ -274,6 +305,20 @@ check_host_status() {
     fi
 }
 
+unset_forced_down_hosts() {
+    for host in $(openstack compute service list --service nova-compute \
+                  -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
+    do
+        # TODO (r-mibu): make sample inspector use keystone v3 api
+        OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \
+        python ./nova_force_down.py $host --unset
+    done
+
+    echo "waiting disabled compute host back to be enabled..."
+    wait_until 'openstack compute service list --service nova-compute
+                -f value -c State | grep -q down' 240 5
+}
+
 cleanup() {
     set +e
     echo "cleanup..."
@@ -281,10 +326,10 @@ cleanup() {
     stop_inspector
     stop_consumer
 
-    echo "waiting disabled compute host back to be enabled..."
-    python ./nova_force_down.py "$COMPUTE_HOST" --unset
-    sleep 240
-    check_host_status "UP"
+    unset_forced_down_hosts
+    # TODO: We need to make sure the target compute host is back to IP
+    #       reachable. wait_ping() will be added by tojuvone .
+    sleep 110
     scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
 
     openstack $as_doctor_user server list | grep -q " $VM_NAME " && openstack $as_doctor_user server delete "$VM_NAME"
@@ -312,6 +357,7 @@ cleanup() {
 # Main process
 
 echo "Note: doctor/tests/run.sh has been executed."
+git log --oneline -1 || true   # ignore even you don't have git installed
 
 trap cleanup EXIT