ci/cyclicTestTrigger.sh

   1 #!/bin/bash
   2
   3 #############################################################
   4 ## This script defines the functions present in
   5 ## test_kvmfornfv.sh interface.These will launch ubuntu
   6 ## docker container runs cyclictest through yardstick
   7 ## and verifies the test results.
   8 ############################################################
   9
  10 HOST_IP=$1
  11 testTime=$2
  12 testType=$3
  13 testName=$4
  14
  15 source $WORKSPACE/ci/envs/utils.sh
  16 source $WORKSPACE/ci/envs/host-config
  17 KERNELRPM_VERSION=$( getKernelVersion )
  18 QEMURPM_VERSION=$( getQemuVersion )
  19
  20 if [ -z ${KERNELRPM_VERSION} ];then
  21    echo "Kernel RPM not found in build_output Directory"
  22    exit 1
  23 fi
  24 if [ -z ${QEMURPM_VERSION} ];then
  25    echo "QEMU RPM not found in build_output Directory"
  26    exit 1
  27 fi
  28
  29 #calculating and verifying sha512sum of the guestimage.
  30 function verifyGuestImage {
  31    scp $WORKSPACE/build_output/guest1.sha512 root@${HOST_IP}:/root/images
  32    checksum=$(sudo ssh root@${HOST_IP} "cd /root/images/ && sha512sum -c guest1.sha512 | awk '{print \$2}'")
  33    if [ "$checksum" != "OK" ]; then
  34       echo "Something wrong with the image, please verify"
  35       return 1
  36    fi
  37 }
  38
  39 #Verifying the availability of the host after reboot
  40 function connect_host {
  41    n=0
  42    while [ $n -lt 25 ]; do
  43       host_ping_test="ping -c 1 ${HOST_IP}"
  44       eval $host_ping_test &> /dev/null
  45       if [ ${?} -ne 0 ] ; then
  46          sleep 10
  47          echo "host machine is still under reboot..trying to connect...."
  48          n=$(($n+1))
  49       else
  50          echo "resuming the execution of test cases....."
  51          #Waiting for ssh to be available for the host machine.
  52          sleep 30
  53          break
  54       fi
  55       if [ $n == 24 ];then
  56          echo "Host machine unable to boot-up!"
  57          exit 1
  58       fi
  59    done
  60 }
  61
  62 #Updating the pod.yaml file with HOST_IP,kvmfornfv_cyclictest_idle_idle.yaml with loops and interval
  63 function updateYaml {
  64    cd $WORKSPACE/tests/
  65    sed -ri "s/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/${HOST_IP}/" pod.yaml
  66    sed -ri "s/loops: [0-9]*/loops: ${testTime}/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
  67    sed -ri "0,/interval: [0-9]*/s//interval: 1000/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
  68    cp kvmfornfv_cyclictest_hostenv_guestenv.yaml kvmfornfv_cyclictest_${testName}.yaml
  69    sed -ri "s/tc: \"kvmfornfv_cyclictest-node-context\"/tc: \"kvmfornfv_cyclictest_${testName}\"/" kvmfornfv_cyclictest_${testName}.yaml
  70    case $testName in
  71
  72        idle_idle)
  73                 ;;
  74        cpustress_idle)
  75                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
  76                       sed -i '/host-setup0.sh/,/host-setup1.sh/ d' kvmfornfv_cyclictest_${testName}.yaml
  77                       ;;
  78        memorystress_idle)
  79                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
  80                       sed -i '/host-setup0.sh/,/host-setup1.sh/ d' kvmfornfv_cyclictest_${testName}.yaml
  81                       ;;
  82        iostress_idle)
  83                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
  84                       sed -i '/host-setup0.sh/,/host-setup1.sh/ d' kvmfornfv_cyclictest_${testName}.yaml
  85                       ;;
  86        idle_cpustress)
  87                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
  88                       sed -i '/host-setup0.sh/,/host-setup1.sh/ d' kvmfornfv_cyclictest_${testName}.yaml
  89                       ;;
  90        idle_memorystress)
  91                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
  92                       sed -i '/host-setup0.sh/,/host-setup1.sh/ d' kvmfornfv_cyclictest_${testName}.yaml
  93                       ;;
  94        idle_iostress)
  95                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
  96                       sed -i '/host-setup0.sh/,/host-setup1.sh/ d' kvmfornfv_cyclictest_${testName}.yaml
  97                       ;;
  98         *)
  99           echo "Incorrect test environment: $testName"
 100           exit 1
 101           ;;
 102     esac
 103 }
 104
 105 #cleaning the environment after executing the test through yardstick.
 106 function env_clean {
 107     container_id=`sudo docker ps -a | grep kvmfornfv_${testType} |awk '{print \$1}'|sed -e 's/\r//g'`
 108     sudo docker stop ${container_id}
 109     sudo docker rm ${container_id}
 110     sudo ssh root@${HOST_IP} "rm -rf /root/workspace/*"
 111     sudo ssh root@${HOST_IP} "pid=\$(ps aux | grep 'qemu' | awk '{print \$2}' | head -1); echo \$pid |xargs kill"
 112     sudo rm -rf /tmp/kvmtest-${testType}*
 113 }
 114
 115 #Cleaning the latest kernel changes on host after executing the test.
 116 function host_clean {
 117     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
 118     sudo ssh root@${HOST_IP} "rm -rf /boot/initramfs-${KERNELRPM_VERSION}*.img"
 119     sudo ssh root@${HOST_IP} "grub2-mkconfig -o /boot/grub2/grub.cfg"
 120     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'qemu-${QEMURPM_VERSION}'| awk '{print \$1}'); rpm -ev \$rpm"
 121     sudo ssh root@${HOST_IP} "reboot"
 122 }
 123
 124 function cleanup {
 125    output=$1
 126    env_clean
 127    host_clean
 128    if [ $output != 0 ];then
 129       echo "Yardstick Failed.Please check cyclictest.sh"
 130       return 1
 131    else
 132       return 0
 133    fi
 134 }
 135
 136 #Creating a docker image with yardstick installed and Verify the results of cyclictest
 137 function runCyclicTest {
 138    container_id=`sudo docker ps -a | grep kvmfornfv_${testType} |awk '{print \$1}'|sed -e 's/\r//g'`
 139    echo "$container_id"
 140    if [ -z "$container_id" ]; then
 141       docker_image_dir=$WORKSPACE/docker_image_build
 142       ( cd ${docker_image_dir}; sudo docker build  -t kvmfornfv:latest --no-cache=true . )
 143       if [ ${?} -ne 0 ] ; then
 144          echo  "Docker image build failed"
 145          id=$(sudo docker ps -a  | head  -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id
 146          exit 1
 147       fi
 148       time_stamp=$(date +%Y%m%d%H%M%S)
 149       volume=/tmp/kvmtest-${testType}-${time_stamp}
 150       mkdir -p $volume/{image,rpm,scripts}
 151       #copying required files to run yardstick cyclic testcase
 152       cp $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
 153       cp $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm ${volume}/rpm
 154       cp -r $WORKSPACE/ci/envs/* ${volume}/scripts
 155       cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
 156       cp -r $WORKSPACE/tests/pod.yaml ${volume}/scripts
 157       #Launching ubuntu docker container to run yardstick
 158       container_id=`sudo docker run -i -d -v ${volume}:/opt --net=host --name kvmfornfv_${testType} \
 159       kvmfornfv:latest`
 160       sudo docker exec -i $container_id /bin/bash -c "cd /opt/scripts && ls; ./cyclictest.sh $testType $testName"
 161       cyclictest_output=$?
 162    else
 163       cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
 164       sudo docker exec -i $container_id /bin/bash -c "cd /opt/scripts && ls; ./cyclictest.sh $testType $testName"
 165       cyclictest_output=$?
 166    fi
 167
 168    if [ "$testName" == "iostress_idle" ];then
 169       copyLogs
 170    fi
 171
 172    #Verifying the results of cyclictest
 173    if [ "$testType" == "verify" ];then
 174       result=`grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
 175
 176       if [ -z "${result}" ] && [ "${cyclictest_output}" == 0 ]; then
 177          echo "####################################################"
 178          echo " "
 179          echo `grep -o '"data":[^}]*' ${volume}/yardstick.out | awk -F '{' '{print $2}'`
 180          echo ""
 181          echo "####################################################"
 182          return 0
 183       else
 184          echo "Testcase failed"
 185          echo `grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
 186          return 1
 187       fi
 188    fi
 189 }