X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=tests%2Frun.sh;h=b5c56872a0c9e37fbb284e203897ed247875a743;hb=dfb3c066a1c325ba7ca028be98d9ec04af994f48;hp=855ecfc6a2d95c9e211229153fd9ead2786ee3d9;hpb=5d9c24fd28bcc02243306a8c96d0c68809523343;p=doctor.git

diff --git a/tests/run.sh b/tests/run.sh
index 855ecfc6..b5c56872 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -21,28 +21,36 @@ VM_BASENAME=doctor_vm
 VM_FLAVOR=m1.tiny
 #if VM_COUNT set, use that instead
 VM_COUNT=${VM_COUNT:-1}
+NET_NAME=doctor_net
+NET_CIDR=192.168.168.0/24
 ALARM_BASENAME=doctor_alarm
-INSPECTOR_PORT=12345
 CONSUMER_PORT=12346
 DOCTOR_USER=doctor
 DOCTOR_PW=doctor
 DOCTOR_PROJECT=doctor
-#TODO: change back to `_member_` when JIRA DOCTOR-55 is done
-DOCTOR_ROLE=admin
-PROFILER_TYPE=${PROFILER_TYPE:-none}
+DOCTOR_ROLE=_member_
+PROFILER_TYPE=${PROFILER_TYPE:-poc}
+PYTHON_ENABLE=${PYTHON_ENABLE:-false}
 
 TOP_DIR=$(cd $(dirname "$0") && pwd)
 
 as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW
-                --os-tenant-name $DOCTOR_PROJECT"
+                --os-project-name $DOCTOR_PROJECT --os-tenant-name $DOCTOR_PROJECT"
+# NOTE: ceilometer command still requires '--os-tenant-name'.
+#ceilometer="ceilometer ${as_doctor_user/--os-project-name/--os-tenant-name}"
+ceilometer="ceilometer $as_doctor_user"
+as_admin_user="--os-username admin --os-project-name $DOCTOR_PROJECT
+               --os-tenant-name $DOCTOR_PROJECT"
 
+upper_constraints="https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/ocata"
+pip_install="pip install -c${upper_constraints}"
 
 # Functions
 
 get_compute_host_info() {
-    # get computer host info which first VM boot in
-    COMPUTE_HOST=$(openstack $as_doctor_user server show ${VM_BASENAME}1 |
-                   grep "OS-EXT-SRV-ATTR:host" | awk '{ print $4 }')
+    # get computer host info which first VM boot in as admin user
+    COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 |
+                   grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }')
     compute_host_in_undercloud=${COMPUTE_HOST%%.*}
     die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname"
 
@@ -104,23 +112,31 @@ register_image() {
 
 create_test_user() {
     openstack project list | grep -q " $DOCTOR_PROJECT " || {
-        openstack project create "$DOCTOR_PROJECT"
+        openstack project create --description "Doctor Project" \
+                                 "$DOCTOR_PROJECT"
     }
     openstack user list | grep -q " $DOCTOR_USER " || {
         openstack user create "$DOCTOR_USER" --password "$DOCTOR_PW" \
                               --project "$DOCTOR_PROJECT"
     }
-    openstack user role list "$DOCTOR_USER" --project "$DOCTOR_PROJECT" \
-    | grep -q " $DOCTOR_ROLE " || {
+    openstack role show "$DOCTOR_ROLE" | grep -q " $DOCTOR_ROLE " || {
+        openstack role create "$DOCTOR_ROLE"
+    }
+    openstack role assignment list --user "$DOCTOR_USER" \
+    --project "$DOCTOR_PROJECT" --names | grep -q " $DOCTOR_ROLE " || {
         openstack role add "$DOCTOR_ROLE" --user "$DOCTOR_USER" \
                            --project "$DOCTOR_PROJECT"
     }
+    openstack role assignment list --user admin --project "$DOCTOR_PROJECT" \
+    --names | grep -q " admin " || {
+        openstack role add admin --user admin --project "$DOCTOR_PROJECT"
+    }
     # tojuvone: openstack quota show is broken and have to use nova
     # https://bugs.launchpad.net/manila/+bug/1652118
     # Note! while it is encouraged to use openstack client it has proven
     # quite buggy.
     # QUOTA=$(openstack quota show $DOCTOR_PROJECT)
-    DOCTOR_QUOTA=$(nova quota-show --tenant DOCTOR_PROJECT)
+    DOCTOR_QUOTA=$(nova quota-show --tenant $DOCTOR_PROJECT)
     # We make sure that quota allows number of instances and cores
     OLD_INSTANCE_QUOTA=$(echo "${DOCTOR_QUOTA}" | grep " instances " | \
                          awk '{print $4}')
@@ -136,28 +152,55 @@ create_test_user() {
     fi
 }
 
+remove_test_user() {
+    openstack project list | grep -q " $DOCTOR_PROJECT " && {
+        openstack role assignment list --user admin \
+        --project "$DOCTOR_PROJECT" --names | grep -q " admin " && {
+            openstack role remove admin --user admin --project "$DOCTOR_PROJECT"
+        }
+        openstack user list | grep -q " $DOCTOR_USER " && {
+            openstack role assignment list --user "$DOCTOR_USER" \
+            --project "$DOCTOR_PROJECT" --names | grep -q " $DOCTOR_ROLE " && {
+                openstack role remove "$DOCTOR_ROLE" --user "$DOCTOR_USER" \
+                --project "$DOCTOR_PROJECT"
+            }
+            openstack user delete "$DOCTOR_USER"
+        }
+        openstack project delete "$DOCTOR_PROJECT"
+    }
+}
+
 boot_vm() {
     # test VM done with test user, so can test non-admin
+
+    if ! openstack $as_doctor_user network show $NET_NAME; then
+        openstack $as_doctor_user network create $NET_NAME
+    fi
+    if ! openstack $as_doctor_user subnet show $NET_NAME; then
+        openstack $as_doctor_user subnet create $NET_NAME \
+            --network $NET_NAME --subnet-range $NET_CIDR --no-dhcp
+    fi
+    net_id=$(openstack $as_doctor_user network show $NET_NAME -f value -c id)
+
     servers=$(openstack $as_doctor_user server list)
     for i in `seq $VM_COUNT`; do
         echo "${servers}" | grep -q " $VM_BASENAME$i " && continue
         openstack $as_doctor_user server create --flavor "$VM_FLAVOR" \
-                            --image "$IMAGE_NAME" \
-                            "$VM_BASENAME$i"
+            --image "$IMAGE_NAME" --nic net-id=$net_id "$VM_BASENAME$i"
     done
     sleep 1
 }
 
 create_alarm() {
     # get vm_id as test user
-    alarm_list=$(ceilometer $as_doctor_user alarm-list)
+    alarm_list=$($ceilometer alarm-list)
     vms=$(openstack $as_doctor_user server list)
     for i in `seq $VM_COUNT`; do
         echo "${alarm_list}" | grep -q " $ALARM_BASENAME$i " || {
             vm_id=$(echo "${vms}" | grep " $VM_BASENAME$i " | awk '{print $2}')
             # TODO(r-mibu): change notification endpoint from localhost to the
             # consumer. IP address (functest container).
-            ceilometer $as_doctor_user alarm-event-create \
+            $ceilometer alarm-event-create \
                        --name "$ALARM_BASENAME$i" \
                        --alarm-action "http://localhost:$CONSUMER_PORT/failure" \
                        --description "VM failure" \
@@ -171,17 +214,6 @@ create_alarm() {
      done
 }
 
-start_monitor() {
-    pgrep -f "python monitor.py" && return 0
-    sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
-        "http://127.0.0.1:$INSPECTOR_PORT/events" > monitor.log 2>&1 &
-}
-
-stop_monitor() {
-    pgrep -f "python monitor.py" || return 0
-    sudo kill $(pgrep -f "python monitor.py")
-}
-
 start_consumer() {
     pgrep -f "python consumer.py" && return 0
     python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 &
@@ -253,9 +285,12 @@ inject_failure() {
     echo "disabling network of compute host [$COMPUTE_HOST] for 3 mins..."
     cat > disable_network.sh << 'END_TXT'
 #!/bin/bash -x
-dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $7}')
-[[ -n "$dev" ]] || dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $5}')
 sleep 1
+if [ -n "@INTERFACE_NAME@" ]; then
+    dev=@INTERFACE_NAME@
+else
+    dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}')
+fi
 sudo ip link set $dev down
 echo "doctor set link down at" $(date "+%s.%N")
 sleep 180
@@ -263,6 +298,7 @@ sudo ip link set $dev up
 sleep 1
 END_TXT
     sed -i -e "s/@COMPUTE_IP@/$COMPUTE_IP/" disable_network.sh
+    sed -i -e "s/@INTERFACE_NAME@/$INTERFACE_NAME/" disable_network.sh
     chmod +x disable_network.sh
     scp $ssh_opts_cpu disable_network.sh "$COMPUTE_USER@$COMPUTE_IP:"
     ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
@@ -287,8 +323,11 @@ calculate_notification_time() {
     wait_consumer 60
     #keep 'at' as the last keyword just before the value, and
     #use regex to get value instead of the fixed column
+    if [ ! -f monitor.log ]; then
+        scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" .
+    fi
     detected=$(grep "doctor monitor detected at" monitor.log |\
-               sed -e "s/^.* at //")
+               sed -e "s/^.* at //" | tail -1)
     notified=$(grep "doctor consumer notified at" consumer.log |\
                sed -e "s/^.* at //" | tail -1)
 
@@ -300,19 +339,6 @@ calculate_notification_time() {
         }'
 }
 
-wait_ping() {
-    local interval=5
-    local rounds=$(($1 / $interval))
-    for i in `seq $rounds`; do
-        ping -c 1 "$COMPUTE_IP"
-        if [[ $? -ne 0 ]] ; then
-            sleep $interval
-            continue
-        fi
-        return 0
-    done
-}
-
 check_host_status() {
     # Check host related to first Doctor VM is in wanted state
     # $1    Expected state
@@ -340,25 +366,36 @@ check_host_status() {
 }
 
 unset_forced_down_hosts() {
-    for host in $(openstack compute service list --service nova-compute \
-                  -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
+    # for debug
+    openstack compute service list --service nova-compute
+
+    downed_computes=$(openstack compute service list --service nova-compute \
+                      -f value -c Host -c State | grep ' down$' \
+                      | sed -e 's/ *down$//')
+    echo "downed_computes: $downed_computes"
+    for host in $downed_computes
     do
-        # TODO (r-mibu): make sample inspector use keystone v3 api
-        OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \
-        python ./nova_force_down.py $host --unset
+        # TODO(r-mibu): use openstack client
+        #openstack compute service set --up $host nova-compute
+        nova service-force-down --unset $host nova-compute
     done
 
     echo "waiting disabled compute host back to be enabled..."
     wait_until 'openstack compute service list --service nova-compute
                 -f value -c State | grep -q down' 240 5
+
+    for host in $downed_computes
+    do
+        # TODO(r-mibu): improve 'get_compute_ip_from_hostname'
+        get_compute_ip_from_hostname $host
+        wait_until "! ping -c 1 $COMPUTE_IP" 120 5
+    done
 }
 
 collect_logs() {
-    unset_forced_down_hosts
-    # TODO: We need to make sure the target compute host is back to IP
-    #       reachable. wait_ping() will be added by tojuvone .
-    sleep 110
-    scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
+    if [[ -n "$COMPUTE_IP" ]];then
+        scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
+    fi
 
     # TODO(yujunz) collect other logs, e.g. nova, aodh
 }
@@ -386,22 +423,20 @@ run_profiler() {
         export DOCTOR_PROFILER_T09=$(python -c \
           "print(int(($notified-$relative_start)*1000))")
 
-        python profiler-poc.py >doctor_profiler.log 2>&1
+        python profiler_poc.py > doctor_profiler.log 2>&1
     fi
 }
 
 cleanup() {
     set +e
     echo "cleanup..."
-    stop_monitor
     stop_inspector
     stop_consumer
 
     unset_forced_down_hosts
+    stop_monitor
+    collect_logs
 
-    wait_ping 120
-
-    scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
     vms=$(openstack $as_doctor_user server list)
     vmstodel=""
     for i in `seq $VM_COUNT`; do
@@ -409,12 +444,15 @@ cleanup() {
         vmstodel+=" $VM_BASENAME$i"
     done
     [[ $vmstodel ]] && openstack $as_doctor_user server delete $vmstodel
-    alarm_list=$(ceilometer $as_doctor_user alarm-list)
+    alarm_list=$($ceilometer alarm-list)
     for i in `seq $VM_COUNT`; do
         alarm_id=$(echo "${alarm_list}" | grep " $ALARM_BASENAME$i " |
                    awk '{print $2}')
-        [ -n "$alarm_id" ] && ceilometer $as_doctor_user alarm-delete "$alarm_id"
+        [ -n "$alarm_id" ] && $ceilometer alarm-delete "$alarm_id"
     done
+    openstack $as_doctor_user subnet delete $NET_NAME
+    sleep 1
+    openstack $as_doctor_user network delete $NET_NAME
     sleep 1
 
     image_id=$(openstack image list | grep " $IMAGE_NAME " | awk '{print $2}')
@@ -423,25 +461,59 @@ cleanup() {
     if [[ "$use_existing_image" == false ]] ; then
         [ -n "$image_id" ] && openstack image delete "$image_id"
     fi
-    openstack role remove "$DOCTOR_ROLE" --user "$DOCTOR_USER" \
-                              --project "$DOCTOR_PROJECT"
-    openstack project delete "$DOCTOR_PROJECT"
-    openstack user delete "$DOCTOR_USER"
+
+    remove_test_user
 
     cleanup_installer
     cleanup_inspector
+    cleanup_monitor
+
+    # NOTE: Temporal log printer.
+    for f in $(find . -name '*.log')
+    do
+        echo
+        echo "[$f]"
+        sed -e 's/^/ | /' $f
+        echo
+    done
+}
+
+setup_python_packages() {
+    pip freeze |grep -i flask\= > /dev/null || sudo ${pip_install} flask
+    command -v openstack || sudo ${pip_install} python-openstackclient
+    command -v ceilometer || sudo ${pip_install} python-ceilometerclient
+    command -v congress || sudo ${pip_install} python-congressclient
 }
 
 # Main process
 
+if [[ $PYTHON_ENABLE == [Tt]rue ]]; then
+    which tox || sudo ${pip_install} tox
+    if [ -f /usr/bin/apt-get ]; then
+        sudo apt-get install -y python3-dev
+    elif [ -f /usr/bin/yum ] ; then
+        sudo yum install -y python3-devel
+    fi
+
+    cd $TOP_DIR
+    echo "executing tox..."
+    tox
+    exit $?
+fi
+
 echo "Note: doctor/tests/run.sh has been executed."
 git log --oneline -1 || true   # ignore even you don't have git installed
 
 trap cleanup EXIT
 
+setup_python_packages
+
 source $TOP_DIR/functions-common
 source $TOP_DIR/lib/installer
 source $TOP_DIR/lib/inspector
+source $TOP_DIR/lib/monitor
+
+rm -f *.log
 
 setup_installer
 
@@ -471,6 +543,7 @@ echo "injecting host failure..."
 inject_failure
 
 check_host_status "(DOWN|UNKNOWN)" 60
+unset_forced_down_hosts
 calculate_notification_time
 collect_logs
 run_profiler