From: Jiang, Yunhong <yunhong.jiang@intel.com>
Date: Wed, 9 Aug 2017 05:38:24 +0000 (+0000)
Subject: Merge "Add Livemigration testcase for KVMFORNFV"
X-Git-Tag: opnfv-5.0.0~7
X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=a074fc668a5b2dd02f5378895c2d36043a5603c5;hp=2dd9dea0edaf1157e4cb914240821d22fe1ca5b0;p=kvmfornfv.git

Merge "Add Livemigration testcase for KVMFORNFV"
---

diff --git a/ci/cyclicTestTrigger.sh b/ci/cyclicTestTrigger.sh
index 7f606b0b6..4bb8fe5d4 100755
--- a/ci/cyclicTestTrigger.sh
+++ b/ci/cyclicTestTrigger.sh
@@ -139,15 +139,15 @@ function cleanup {
    fi
 }
 
-#environment setup for executing packet forwarding test cases
-function setUpPacketForwarding {
-   #copying required files to run packet forwarding test cases
+function nodeSetup {
+   #copying required files to run kvmfornfv testcases
    ssh root@$HOST_IP "mkdir -p /root/workspace/image"
    ssh root@$HOST_IP "mkdir -p /root/workspace/rpm"
    ssh root@$HOST_IP "mkdir -p /root/workspace/scripts"
    #Copying the host configuration scripts on to host
    scp -r $WORKSPACE/ci/envs/* root@$HOST_IP:/root/workspace/scripts
    scp -r $WORKSPACE/tests/vsperf.conf* root@$HOST_IP:/root/workspace/scripts
+   scp -r $WORKSPACE/tests/pod.yaml root@$HOST_IP:/root/workspace/scripts
    scp -r $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
    scp -r $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
    scp -r $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
@@ -158,6 +158,12 @@ function setUpPacketForwarding {
    sleep 10
 }
 
+#environment setup for executing packet forwarding test cases
+function setUpPacketForwarding {
+   echo "Copying required files to execute packet forwarding test case"
+   nodeSetup
+}
+
 #executing packet forwarding test cases
 function runPacketForwarding {
    testType=$1
@@ -220,3 +226,24 @@ function runCyclicTest {
       cleanup $cyclictest_output
    fi
 }
+function runLiveMigration {
+   test_env=$1
+   if [ ${test_env} == "peer-peer" ];then
+      echo "live migration is not implemented for peer to peer"
+   else
+      echo "In runLiveMigration Function"
+      echo "Copying required files to execute live migration"
+      nodeSetup
+      connect_host
+      sleep 15
+      echo " Displaying the number of huge pages on node"
+      ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
+      echo " Displaying the free huge pages on node"
+      ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
+      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup1.sh"
+      echo "Setting up ovs-dpdk on the host"
+      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./setup_ovsdpdk.sh"
+      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-install-qemu.sh"
+      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-run-livemigration.sh"
+  fi
+}
diff --git a/ci/envs/host-run-livemigration.sh b/ci/envs/host-run-livemigration.sh
new file mode 100755
index 000000000..a089ad482
--- /dev/null
+++ b/ci/envs/host-run-livemigration.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+##############################################################################
+##Copyright (c) 2015 Intel Corp.
+##
+## All rights reserved. This program and the accompanying materials
+## are made available under the terms of the Apache License, Version 2.0
+## which accompanies this distribution, and is available at
+## http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+source utils.sh
+source host-config
+
+HOST_IP=$( getHostIP )
+#source qmp-sock for conmunication with qemu
+qmp_sock_src="/tmp/qmp-sock-src"
+#destination qmp-sock for conmunication with qemu, only for local live migration
+qmp_sock_dst="/tmp/qmp-sock-dst"
+
+VHOSTPATH1='/usr/local/var/run/openvswitch/vhost-user1'
+VHOSTPATH2='/usr/local/var/run/openvswitch/vhost-user2'
+
+VHOSTPATH3='/usr/local/var/run/openvswitch/vhost-user3'
+VHOSTPATH4='/usr/local/var/run/openvswitch/vhost-user4'
+
+MACADDRESS1='52:54:00:12:34:56'
+MACADDRESS2='54:54:00:12:34:56'
+
+#destination host ip address
+incoming_ip=0
+migrate_port=4444
+max_down_time=10
+
+
+OVSLOGFILE='/var/log/openvswitch/ovs-vswitchd.log'
+
+function run_qemusrc() {
+  $qemu -enable-kvm -cpu host -smp ${guest_cpus} -chardev socket,id=char1,path=$VHOSTPATH1 \
+        -netdev type=vhost-user,id=net1,chardev=char1,vhostforce \
+        -device virtio-net-pci,netdev=net1,mac=$MACADDRESS1 \
+        -chardev socket,id=char2,path=$VHOSTPATH2 \
+        -netdev type=vhost-user,id=net2,chardev=char2,vhostforce \
+        -device virtio-net-pci,netdev=net2,mac=$MACADDRESS2 -m 1024 -mem-path /dev/hugepages \
+        -mem-prealloc -realtime mlock=on -monitor unix:${qmp_sock_src},server,nowait \
+        -balloon virtio -drive file=/root/guest1.qcow2 -vnc :1 &
+  if [ ${?} -ne 0 ] ; then
+    echo "Qemu Source not started"
+    exit 1
+  fi
+}
+
+function run_qemulisten() {
+  $qemu -enable-kvm -cpu host -smp ${guest_cpus} -chardev socket,id=char1,path=$VHOSTPATH3 \
+  -netdev type=vhost-user,id=net1,chardev=char1,vhostforce \
+  -device virtio-net-pci,netdev=net1,mac=$MACADDRESS1 \
+  -chardev socket,id=char2,path=$VHOSTPATH4 \
+  -netdev type=vhost-user,id=net2,chardev=char2,vhostforce \
+  -device virtio-net-pci,netdev=net2,mac=$MACADDRESS2 -m 1024 -mem-path /dev/hugepages \
+  -mem-prealloc -realtime mlock=on -monitor unix:${qmp_sock_dst},server,nowait \
+  -balloon virtio -drive file=/root/guest1.qcow2 -incoming tcp:${incoming_ip}:${migrate_port} -vnc :3 &
+  if [ ${?} -ne 0 ] ; then
+    echo "Qemu Standby not started"
+    exit 1
+  fi
+}
+
+function do_migration() {
+
+  local src=$1
+  local dst=$2
+#with no speed limit
+  echo "migrate_set_speed 0" |nc -U $src
+#set the expected max downtime
+  echo "migrate_set_downtime ${max_down_time}" |nc -U $src
+#start live migration
+  echo "migrate -d tcp:${incoming_ip}:${migrate_port}" |nc -U $src
+#wait until live migration completed
+  status=""
+  while [  "${status}" == ""  ]
+  do
+     status=`echo "info migrate" | nc -U $src |grep completed | cut -d: -f2`
+     echo ${status}
+     sleep 1;
+  done
+#get the related data
+ status=`echo "info migrate" | nc -U $src |grep completed | cut -d: -f2`
+ total_time=`echo "info migrate" | nc -U $src |grep "total time" | cut -d: -f2`
+ down_time=`echo "info migrate" | nc -U $src |grep "downtime" | cut -d: -f2`
+
+#print detail information
+ echo "info migrate" | nc -U $src
+ echo "quit" | nc -U $src
+ echo "quit" | nc -U $dst
+ sleep 5
+ echo "Migration executed successfully"
+}
+echo "Running Qemu Source"
+run_qemusrc
+sleep 60
+echo "Running Qemu listen"
+run_qemulisten
+sleep 60
+do_migration $qmp_sock_src $qmp_sock_dst
+if [ ${?} -ne 0 ] ; then
+   echo "Migration Failed"
+   exit 1
+fi
diff --git a/ci/envs/setup_ovsdpdk.sh b/ci/envs/setup_ovsdpdk.sh
new file mode 100755
index 000000000..bd26dfbf6
--- /dev/null
+++ b/ci/envs/setup_ovsdpdk.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#############################################################################
+#Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+source host-config
+OVSLOGFILE="/usr/local/var/run/openvswitch/ovs-vswitchd.log"
+HOME='/home/jenkins'
+VSPERF="${HOME}/vswitchperf"
+
+function install_ovsdpdk() {
+ #Installing ovs dpdk using vsperf environment
+ echo "Installing ovs dpdk using vsperf environment"
+ install_vsperf
+ setup_ovsdpdk
+}
+function install_vsperf() {
+ echo "Installing vsperf....."
+ ( cd $VSPERF/systems ; ./build_base_machine.sh )
+ if [ ${?} -ne 0 ]; then
+    echo "Execution of build_base_machine.sh failed"
+    exit 1
+ fi
+}
+function setup_ovsdpdk() {
+ sudo mkdir -p /usr/local/var/run/openvswitch
+ sudo mkdir -p /usr/local/etc/openvswitch
+ sudo modprobe openvswitch
+
+ #Kill the ovsswitch and ovsdbserver
+ cd $VSPERF/src/ovs/ovs
+ ps aux | grep 'ovsdb-server.pid' | awk '{print $2}' | head -1 | xargs kill -SIGTERM
+ ps aux | grep 'ovs-vswitchd.pid' | awk '{print $2}' | head -1 | xargs kill -SIGTERM
+ kill -SIGTERM 39424
+
+ rm -f /usr/local/var/run/openvswitch/vhost-user*
+ rm -f /usr/local/etc/openvswitch/conf.db
+
+ #Start database server
+ echo "Start ovs database server"
+ export DB_SOCK=/usr/local/var/run/openvswitch/db.sock
+ cd $VSPERF/src/ovs/ovs/ovsdb
+ sudo ./ovsdb-tool create /usr/local/etc/openvswitch/conf.db $VSPERF/src/ovs/ovs/vswitchd/vswitch.ovsschema
+ if [ ${?} -ne 0 ] ; then
+    echo "Creation of db and schema files failed"
+    exit 1
+ fi
+ sudo ./ovsdb-server --remote=punix:$DB_SOCK --remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
+ if [ ${?} -ne 0 ] ; then
+    echo "Starting of DB server failed"
+    exit 1
+ fi
+ #Start OVS
+ echo "Start OVS"
+ cd $VSPERF/src/ovs/ovs/utilities
+ sudo ./ovs-vsctl --no-wait init
+ sudo ./ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0xf
+ sudo ./ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=1024
+ sudo ./ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+ 
+ echo "Setting Huge pages on Node0"
+ node0_pages=10
+ node0_dir="/sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages"
+ node0_pages+=`cat $node0_dir`
+ echo ${node0_pages} > ${node0_dir}
+
+ echo "Setting Huge pages on Node1"
+ node1_pages=10
+ node1_dir="/sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
+ node1_pages+=`cat $node1_dir`
+ echo ${node1_pages} > ${node1_dir}
+
+ free_hp0=`cat /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/free_hugepages`
+ free_hp1=`cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/free_hugepages`
+ echo "Free Hugepages:${free_hp0}"
+ echo "Free_Hugepages:${free_hp1}"
+
+ cd $VSPERF/src/ovs/ovs/vswitchd
+ sudo ./ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file=$OVSLOGFILE
+ if [ ${?} -ne 0 ] ; then
+    echo "ovs-vswitchd not started"
+    exit 1
+ fi
+ sleep 180
+ #Configure the bridge
+ echo "configure OVS Bridge"
+ cd $VSPERF/src/ovs/ovs/utilities
+ sudo ./ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
+ sudo ./ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuser
+ sudo ./ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuser
+ sudo ./ovs-vsctl add-port ovsbr0 vhost-user3 -- set Interface vhost-user3 type=dpdkvhostuser
+ sudo ./ovs-vsctl add-port ovsbr0 vhost-user4 -- set Interface vhost-user4 type=dpdkvhostuser
+}
+install_ovsdpdk
diff --git a/ci/test_kvmfornfv.sh b/ci/test_kvmfornfv.sh
index 6540b18ca..522d6a1a2 100755
--- a/ci/test_kvmfornfv.sh
+++ b/ci/test_kvmfornfv.sh
@@ -18,6 +18,7 @@ cyclictest_env_verify=("idle_idle" "memorystress_idle") #cyclictest environment
 cyclictest_env_daily=("idle_idle" "cpustress_idle" "memorystress_idle" "iostress_idle")
 cyclictest_result=0 #exit code of cyclictest
 packetforward_result=0 #exit code of packet forward
+lm_env_verify=("peer-peer" "local")
 source $WORKSPACE/ci/envs/host-config
 
 #check if any kernel rpms available for testing
@@ -52,6 +53,25 @@ function packetForward {
       exit 1
    fi
 }
+function liveMigration {
+   #executing live migration test case on the host machine
+   test_env=$1
+   echo "Test Environment ${test_env}"
+   if [ ${test_env} == "peer-peer" ];then
+      echo "live migration is not implemented for peer to peer"
+      livemigration_result=0
+   elif [ ${test_env} == "local" ];then
+      source $WORKSPACE/ci/cyclicTestTrigger.sh $HOST_IP
+      connect_host
+      #Waiting for ssh to be available for the host machine.
+      sleep 20
+      runLiveMigration ${test_env}
+      livemigration_result=$?
+   else
+      echo "Incorrect test environment for live migration"
+      exit 1
+   fi
+}
 
 function getTestParams {
    HOST_IP=$( setHostIP $test_type )
@@ -140,6 +160,11 @@ if [ ${test_type} == "verify" ];then
       done
       #Execution of packet forwarding test cases.
       packetForward
+      for envi in ${lm_env_verify[@]}
+      do
+         echo "Executing Live Migration on the node"
+         liveMigration ${envi}
+      done
    fi
    if [ ${cyclictest_result} -ne 0 ] ||  [ ${packetforward_result} -ne 0 ];then
       echo "Test case FAILED"
@@ -192,6 +217,21 @@ elif [ ${test_type} == "daily" ];then
             echo "Cyclictest case executed SUCCESSFULLY"
             test_exit 0
          fi
+   elif [ ${test_name} == "livemigration" ];then
+         for envi in ${lm_env_verify[@]}
+         do
+         echo "Executing Live Migration on the node"
+         liveMigration ${envi}
+         done
+         sudo ssh root@${HOST_IP} "rm -rf /root/workspace/*"
+         host_clean
+         if [ ${livemigration_result} -ne 0 ] ; then
+            echo "livemigration test case execution FAILED"
+            test_exit 1
+         else
+            echo "livemigration test case executed SUCCESSFULLY"
+            test_exit 0
+         fi
    fi
 elif [ ${test_type} == "merge" ];then
    echo "Test is not enabled for ${test_type}"