ci/cyclicTestTrigger.sh

   1 #!/bin/bash
   2
   3 #############################################################
   4 ## This script defines the functions present in
   5 ## test_kvmfornfv.sh interface.These will launch ubuntu
   6 ## docker container runs cyclictest through yardstick
   7 ## and verifies the test results.
   8 ############################################################
   9
  10 HOST_IP=$1
  11 testTime=$2
  12 testType=$3
  13 testName=$4
  14
  15 source $WORKSPACE/ci/envs/utils.sh
  16 source $WORKSPACE/ci/envs/host-config
  17
  18 checkRPMNames
  19
  20 KERNELRPM_VERSION=$( getKernelVersion )
  21 QEMURPM_VERSION=$( getQemuVersion )
  22
  23 if [ -z ${KERNELRPM_VERSION} ];then
  24    echo "Kernel RPM not found in build_output Directory"
  25    exit 1
  26 fi
  27 if [ -z ${QEMURPM_VERSION} ];then
  28    echo "QEMU RPM not found in build_output Directory"
  29    exit 1
  30 fi
  31
  32 #calculating and verifying sha512sum of the guestimage.
  33 function verifyGuestImage {
  34    scp $WORKSPACE/build_output/guest1.sha512 root@${HOST_IP}:/root/images
  35    checksum=$(sudo ssh root@${HOST_IP} "cd /root/images/ && sha512sum -c guest1.sha512 | awk '{print \$2}'")
  36    if [ "$checksum" != "OK" ]; then
  37       echo "Something wrong with the image, please verify"
  38       return 1
  39    fi
  40 }
  41
  42 #disabling ftrace and collecting the logs to upload to artifact repository.
  43 function ftrace_disable {
  44    sudo ssh root@${HOST_IP} "cd /tmp ;  mv trace.txt cyclictest_${env}.txt"
  45    mkdir -p $WORKSPACE/build_output/log/kernel_trace
  46    scp root@${HOST_IP}:/tmp/cyclictest_${env}.txt $WORKSPACE/build_output/log/kernel_trace/
  47    sudo ssh root@${HOST_IP} "cd /tmp ; rm -rf cyclictest_${env}.txt"
  48 }
  49
  50 #Verifying the availability of the host after reboot
  51 function connect_host {
  52    n=0
  53    while [ $n -lt 25 ]; do
  54       host_ping_test="ping -c 1 ${HOST_IP}"
  55       eval $host_ping_test &> /dev/null
  56       if [ ${?} -ne 0 ] ; then
  57          sleep 10
  58          echo "host machine is still under reboot..trying to connect...."
  59          n=$(($n+1))
  60       else
  61          echo "resuming the execution of test cases....."
  62          #Waiting for ssh to be available for the host machine.
  63          sleep 30
  64          break
  65       fi
  66       if [ $n == 24 ];then
  67          echo "Host machine unable to boot-up!"
  68          exit 1
  69       fi
  70    done
  71 }
  72
  73 #Updating the pod.yaml file with HOST_IP,kvmfornfv_cyclictest_idle_idle.yaml with loops and interval
  74 function updateYaml {
  75    cd $WORKSPACE/tests/
  76    sed -ri "s/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/${HOST_IP}/" pod.yaml
  77    sed -ri "s/loops: [0-9]*/loops: ${testTime}/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
  78    sed -ri "0,/interval: [0-9]*/s//interval: 1000/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
  79    cp kvmfornfv_cyclictest_hostenv_guestenv.yaml kvmfornfv_cyclictest_${testName}.yaml
  80    sed -ri "s/tc: \"kvmfornfv_cyclictest-node-context\"/tc: \"kvmfornfv_cyclictest_${testName}\"/" kvmfornfv_cyclictest_${testName}.yaml
  81    case $testName in
  82
  83        idle_idle)
  84                 ;;
  85        cpustress_idle)
  86                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
  87                       ;;
  88        memorystress_idle)
  89                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
  90                       ;;
  91        iostress_idle)
  92                       sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
  93                       ;;
  94        idle_cpustress)
  95                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
  96                       ;;
  97        idle_memorystress)
  98                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
  99                       ;;
 100        idle_iostress)
 101                       sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
 102                       ;;
 103         *)
 104           echo "Incorrect test environment: $testName"
 105           exit 1
 106           ;;
 107     esac
 108 }
 109
 110 #cleaning the environment after executing the test through yardstick.
 111 function env_clean {
 112     container_id=`sudo docker ps -a | grep kvmfornfv_${testType} |awk '{print \$1}'|sed -e 's/\r//g'`
 113     sudo docker stop ${container_id}
 114     sudo docker rm ${container_id}
 115     sudo ssh root@${HOST_IP} "rm -rf /root/workspace/*"
 116     sudo ssh root@${HOST_IP} "pid=\$(ps aux | grep 'qemu' | awk '{print \$2}' | head -1); echo \$pid |xargs kill"
 117     sudo rm -rf /tmp/kvmtest-${testType}*
 118 }
 119
 120 #Cleaning the latest kernel changes on host after executing the test.
 121 function host_clean {
 122     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
 123     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-devel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
 124     sudo ssh root@${HOST_IP} "rm -rf /boot/initramfs-${KERNELRPM_VERSION}*.img"
 125     sudo ssh root@${HOST_IP} "grub2-mkconfig -o /boot/grub2/grub.cfg"
 126     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'qemu-${QEMURPM_VERSION}'| awk '{print \$1}'); rpm -ev \$rpm"
 127     sudo ssh root@${HOST_IP} "reboot"
 128 }
 129
 130 function cleanup {
 131    output=$1
 132    env_clean
 133    host_clean
 134    if [ $output != 0 ];then
 135       echo "Yardstick Failed.Please check cyclictest.sh"
 136       return 1
 137    else
 138       return 0
 139    fi
 140 }
 141
 142 function nodeSetup {
 143    #copying required files to run kvmfornfv testcases
 144    ssh root@$HOST_IP "mkdir -p /root/workspace/image"
 145    ssh root@$HOST_IP "mkdir -p /root/workspace/rpm"
 146    ssh root@$HOST_IP "mkdir -p /root/workspace/scripts"
 147    #Copying the host configuration scripts on to host
 148    scp -r $WORKSPACE/ci/envs/* root@$HOST_IP:/root/workspace/scripts
 149    scp -r $WORKSPACE/tests/vsperf.conf* root@$HOST_IP:/root/workspace/scripts
 150    scp -r $WORKSPACE/tests/pod.yaml root@$HOST_IP:/root/workspace/scripts
 151    scp -r $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
 152    scp -r $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
 153    scp -r $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
 154    #execute host configuration script for installing kvm built kernel.
 155    ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup0.sh"
 156    ssh root@$HOST_IP "cd /root/workspace/rpm ; rpm -ivh kernel-devel-${KERNELRPM_VERSION}*.rpm"
 157    ssh root@$HOST_IP "reboot"
 158    sleep 10
 159 }
 160
 161 #environment setup for executing packet forwarding test cases
 162 function setUpPacketForwarding {
 163    echo "Copying required files to execute packet forwarding test case"
 164    nodeSetup
 165 }
 166
 167 #executing packet forwarding test cases
 168 function runPacketForwarding {
 169    testType=$1
 170    ssh -t -t root@$HOST_IP "cd /root/workspace/scripts ; sudo scl enable python33 'sh packet_forwarding.sh $testType $QEMURPM_VERSION'"
 171 }
 172
 173 #Creating a docker image with yardstick installed and Verify the results of cyclictest
 174 function runCyclicTest {
 175    ftrace_enable=$1
 176    docker_image_dir=$WORKSPACE/docker_image_build
 177    ( cd ${docker_image_dir}; sudo docker build  -t kvmfornfv:latest --no-cache=true . )
 178    if [ ${?} -ne 0 ] ; then
 179       echo  "Docker image build failed"
 180       id=$(sudo docker ps -a  | head  -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id
 181       exit 1
 182    fi
 183    time_stamp=$(date +%Y%m%d%H%M%S)
 184    volume=/tmp/kvmtest-${testType}-${time_stamp}
 185    mkdir -p $volume/{image,rpm,scripts}
 186    #copying required files to run yardstick cyclic testcase
 187    cp $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
 188    cp $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm ${volume}/rpm
 189    cp -r $WORKSPACE/ci/envs/* ${volume}/scripts
 190    cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
 191    cp -r $WORKSPACE/tests/pod.yaml ${volume}/scripts
 192
 193    #Launching ubuntu docker container to run yardstick
 194    sudo docker run -i -v ${volume}:/opt --net=host --name kvmfornfv_${testType}_${testName} \
 195    kvmfornfv:latest /bin/bash -c "cd /opt/scripts && ls; ./cyclictest.sh $testType $testName"
 196    cyclictest_output=$?
 197
 198    #Disabling ftrace after completion of executing test cases.
 199    if [ ${ftrace_enable} -eq '1' ]; then
 200       ftrace_disable
 201    fi
 202
 203    if [ "$testName" == "memorystress_idle" ];then
 204       copyLogs
 205    fi
 206
 207    #Verifying the results of cyclictest
 208    if [ "$testType" == "verify" ];then
 209       result=`grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
 210
 211       if [ -z "${result}" ]; then
 212          echo "####################################################"
 213          echo " "
 214          echo `grep -o '"data":[^}]*' ${volume}/yardstick.out | awk -F '{' '{print $2}'`
 215          echo ""
 216          echo "####################################################"
 217          cleanup $cyclictest_output
 218       else
 219          echo "Testcase failed"
 220          echo `grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
 221          env_clean
 222          host_clean
 223          return 1
 224       fi
 225    else
 226       cleanup $cyclictest_output
 227    fi
 228 }
 229 function runLiveMigration {
 230    test_env=$1
 231    if [ ${test_env} == "peer-peer" ];then
 232       echo "live migration is not implemented for peer to peer"
 233    else
 234       echo "In runLiveMigration Function"
 235       echo "Copying required files to execute live migration"
 236       nodeSetup
 237       connect_host
 238       sleep 15
 239       echo " Displaying the number of huge pages on node"
 240       ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
 241       echo " Displaying the free huge pages on node"
 242       ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
 243       ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup1.sh"
 244       echo "Setting up ovs-dpdk on the host"
 245       ssh root@$HOST_IP "cd /root/workspace/scripts ; ./setup_ovsdpdk.sh"
 246       ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-install-qemu.sh"
 247       ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-run-livemigration.sh"
 248   fi
 249 }