ci/cyclicTestTrigger.sh

   1 #!/bin/bash
   2
   3 #############################################################
   4 ## This script defines the functions present in
   5 ## test_kvmfornfv.sh interface.These will launch ubuntu
   6 ## docker container runs cyclictest through yardstick
   7 ## and verifies the test results.
   8 ############################################################
   9
  10 HOST_IP=$1
  11 testTime=$2
  12 testType=$3
  13 testName=$4
  14
  15 source $WORKSPACE/ci/envs/utils.sh
  16 source $WORKSPACE/ci/envs/host-config
  17
  18 checkRPMNames
  19
  20 KERNELRPM_VERSION=$( getKernelVersion )
  21 QEMURPM_VERSION=$( getQemuVersion )
  22
  23 if [ -z ${KERNELRPM_VERSION} ];then
  24    echo "Kernel RPM not found in build_output Directory"
  25    exit 1
  26 fi
  27 if [ -z ${QEMURPM_VERSION} ];then
  28    echo "QEMU RPM not found in build_output Directory"
  29    exit 1
  30 fi
  31
  32 #calculating and verifying sha512sum of the guestimage.
  33 function verifyGuestImage {
  34    scp $WORKSPACE/build_output/guest1.sha512 root@${HOST_IP}:/root/images
  35    checksum=$(sudo ssh root@${HOST_IP} "cd /root/images/ && sha512sum -c guest1.sha512 | awk '{print \$2}'")
  36    if [ "$checksum" != "OK" ]; then
  37       echo "Something wrong with the image, please verify"
  38       return 1
  39    fi
  40 }
  41
  42 #disabling ftrace and collecting the logs to upload to artifact repository.
  43 function ftrace_disable {
  44    sudo ssh root@${HOST_IP} "cd /tmp ;  mv trace.txt cyclictest_${env}.txt"
  45    mkdir -p $WORKSPACE/build_output/log/kernel_trace
  46    scp root@${HOST_IP}:/tmp/cyclictest_${env}.txt $WORKSPACE/build_output/log/kernel_trace/
  47    sudo ssh root@${HOST_IP} "cd /tmp ; rm -rf cyclictest_${env}.txt"
  48 }
  49
  50 #Verifying the availability of the host after reboot
  51 function connect_host {
  52    n=0
  53    while [ $n -lt 25 ]; do
  54       host_ping_test="ping -c 1 ${HOST_IP}"
  55       eval $host_ping_test &> /dev/null
  56       if [ ${?} -ne 0 ] ; then
  57          sleep 10
  58          echo "host machine is still under reboot..trying to connect...."
  59          n=$(($n+1))
  60       else
  61          echo "resuming the execution of test cases....."
  62          #Waiting for ssh to be available for the host machine.
  63          sleep 30
  64          break
  65       fi
  66       if [ $n == 24 ];then
  67          echo "Host machine unable to boot-up!"
  68          exit 1
  69       fi
  70    done
  71 }
  72
  73 #Updating the pod.yaml file with HOST_IP,kvmfornfv_cyclictest_idle_idle.yaml with loops and interval
  74 function updateYaml {
  75    cd $WORKSPACE/tests/
  76    sed -ri "s/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/${HOST_IP}/" pod.yaml
  77    sed -ri "s/loops: [0-9]*/loops: ${testTime}/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
  78    sed -ri "0,/interval: [0-9]*/s//interval: 1000/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
  79    cp kvmfornfv_cyclictest_hostenv_guestenv.yaml kvmfornfv_cyclictest_${testName}.yaml
  80    sed -ri "s/tc: \"kvmfornfv_cyclictest-node-context\"/tc: \"kvmfornfv_cyclictest_${testName}\"/" kvmfornfv_cyclictest_${testName}.yaml
  81    case $testName in
  82
  83        idle_idle)
  84                 ;;
  85        cpustress_idle)
  86                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
  87                       ;;
  88        memorystress_idle)
  89                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
  90                       ;;
  91        iostress_idle)
  92                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
  93                       ;;
  94        idle_cpustress)
  95                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
  96                       ;;
  97        idle_memorystress)
  98                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
  99                       ;;
 100        idle_iostress)
 101                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
 102                       ;;
 103         *)
 104           echo "Incorrect test environment: $testName"
 105           exit 1
 106           ;;
 107     esac
 108 }
 109
 110 #cleaning the environment after executing the test through yardstick.
 111 function env_clean {
 112     container_id=`sudo docker ps -a | grep kvmfornfv_${testType} |awk '{print \$1}'|sed -e 's/\r//g'`
 113     sudo docker stop ${container_id}
 114     sudo docker rm ${container_id}
 115     sudo ssh root@${HOST_IP} "rm -rf /root/workspace/*"
 116     sudo ssh root@${HOST_IP} "pid=\$(ps aux | grep 'qemu' | awk '{print \$2}' | head -1); echo \$pid |xargs kill"
 117     sudo rm -rf /tmp/kvmtest-${testType}*
 118 }
 119
 120 #Cleaning the latest kernel changes on host after executing the test.
 121 function host_clean {
 122     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
 123     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-devel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
 124     sudo ssh root@${HOST_IP} "rm -rf /boot/initramfs-${KERNELRPM_VERSION}*.img"
 125     sudo ssh root@${HOST_IP} "grub2-mkconfig -o /boot/grub2/grub.cfg"
 126     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'qemu-${QEMURPM_VERSION}'| awk '{print \$1}'); rpm -ev \$rpm"
 127     sudo ssh root@${HOST_IP} "reboot"
 128 }
 129
 130 function cleanup {
 131    output=$1
 132    env_clean
 133    host_clean
 134    if [ $output != 0 ];then
 135       echo "Yardstick Failed.Please check your testcase"
 136       return 1
 137    else
 138       return 0
 139    fi
 140 }
 141
 142 #environment setup for executing cyclictest and live migration test cases
 143 function setUpEnv {
 144    test=$1
 145    time_stamp=$(date +%Y%m%d%H%M%S)
 146    volume=/tmp/kvmtest-${testType}-${time_stamp}
 147    mkdir -p $volume/{image,rpm,scripts}
 148    #copying required files to run yardstick cyclic testcase
 149    cp $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
 150    cp $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
 151    cp $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm ${volume}/rpm
 152    cp -r $WORKSPACE/ci/envs/* ${volume}/scripts
 153    cp -r $WORKSPACE/tests/pod.yaml ${volume}/scripts
 154    if [ "$test" == "cyclictest" ];then
 155       cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
 156    else
 157       cp -r $WORKSPACE/tests/migrate-node-context.yaml ${volume}
 158    fi
 159 }
 160
 161 #environment setup for executing packet forwarding test cases
 162 function setUpPacketForwarding {
 163    #copying required files to run packet forwarding test cases
 164    ssh root@$HOST_IP "mkdir -p /root/workspace/image"
 165    ssh root@$HOST_IP "mkdir -p /root/workspace/rpm"
 166    ssh root@$HOST_IP "mkdir -p /root/workspace/scripts"
 167    #Copying the host configuration scripts on to host
 168    scp -r $WORKSPACE/ci/envs/* root@$HOST_IP:/root/workspace/scripts
 169    scp -r $WORKSPACE/tests/vsperf.conf* root@$HOST_IP:/root/workspace/scripts
 170    scp -r $WORKSPACE/tests/pod.yaml root@$HOST_IP:/root/workspace/scripts
 171    scp -r $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
 172    scp -r $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
 173    scp -r $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
 174    #execute host configuration script for installing kvm built kernel.
 175    ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup0.sh"
 176    ssh root@$HOST_IP "cd /root/workspace/rpm ; rpm -ivh kernel-devel-${KERNELRPM_VERSION}*.rpm"
 177    ssh root@$HOST_IP "reboot"
 178    sleep 10
 179 }
 180
 181 #executing packet forwarding test cases
 182 function runPacketForwarding {
 183    testType=$1
 184    ssh -t -t root@$HOST_IP "cd /root/workspace/scripts ; sudo scl enable python33 'sh packet_forwarding.sh $testType $QEMURPM_VERSION'"
 185 }
 186
 187 #Creating a docker image with yardstick installed and Verify the results of cyclictest
 188 function runCyclicTest {
 189    ftrace_enable=$1
 190    variable=$2
 191    docker_image_dir=$WORKSPACE/docker_image_build
 192    ( cd ${docker_image_dir}; sudo docker build  -t kvmfornfv:latest --no-cache=true . )
 193    if [ ${?} -ne 0 ] ; then
 194       echo  "Docker image build failed"
 195       id=$(sudo docker ps -a  | head  -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id
 196       exit 1
 197    fi
 198
 199    #setting up the environment for cyclictest
 200    setUpEnv $variable
 201
 202    #Launching ubuntu docker container to run yardstick
 203    sudo docker run -i -v ${volume}:/opt --net=host --name kvmfornfv_${testType}_${testName} \
 204    kvmfornfv:latest /bin/bash -c "cd /opt/scripts && ls; ./cyclictest.sh $testType $testName"
 205    cyclictest_output=$?
 206
 207    #Disabling ftrace after completion of executing test cases.
 208    if [ ${ftrace_enable} -eq '1' ]; then
 209       ftrace_disable
 210    fi
 211
 212    if [ "$testName" == "memorystress_idle" ];then
 213       copyLogs
 214    fi
 215
 216    #Verifying the results of cyclictest
 217    if [ "$testType" == "verify" ];then
 218       result=`grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
 219
 220       if [ -z "${result}" ]; then
 221          echo "####################################################"
 222          echo " "
 223          echo `grep -o '"data":[^}]*' ${volume}/yardstick.out | awk -F '{' '{print $2}'`
 224          echo ""
 225          echo "####################################################"
 226          cleanup $cyclictest_output
 227       else
 228          echo "Testcase failed"
 229          echo `grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
 230          env_clean
 231          host_clean
 232          return 1
 233       fi
 234    else
 235       cleanup $cyclictest_output
 236    fi
 237 }
 238 function runLiveMigration {
 239    echo "In live migration function"
 240    test_env=$1
 241    variable=$2
 242    #Setting up the environment for live migration test case
 243    setUpEnv $variable
 244    #Launching ubuntu docker container to run yardstick
 245    sudo docker run -i -v ${volume}:/opt --net=host --name kvmfornfv_lm_${test_env} \
 246    kvmfornfv:latest /bin/bash -c "cd /opt/scripts && ls; ./lmtest.sh "
 247    lmtest_result=$?
 248    #Verifying the results of livemigration
 249    if [ ${lmtest_result} -ne 0 ];then
 250       env_clean
 251       host_clean
 252       return 1
 253    else
 254       cleanup $lmtest_result
 255    fi
 256 }