ci/build-vsperf.sh

   1 #!/bin/bash
   2 #
   3 # Copyright 2015-2016 Intel Corporation.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #   http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 # VSPERF nightly build execution script
  18
  19 # Usage:
  20 #       build-vsperf.sh job_type
  21 #   where job_type is one of "verify", "merge", "daily"
  22
  23 #
  24 # exit codes
  25 #
  26
  27 EXIT=0
  28 EXIT_TC_FAILED=1
  29 EXIT_SANITY_TC_FAILED=2
  30 EXIT_PYLINT_FAILED=4
  31 EXIT_NO_RESULTS=128
  32 EXIT_NO_TEST_REPORT_LOG_DIR=256
  33
  34 #
  35 # configuration
  36 #
  37
  38 VSPERF_BIN='./vsperf'
  39 LOG_FILE_PREFIX="/tmp/vsperf_build"
  40 DATE=$(date -u +"%Y-%m-%d_%H-%M-%S")
  41 BRANCH=${GIT_BRANCH##*/}
  42 VSPERFENV_DIR="$HOME/vsperfenv"
  43
  44 # CI job specific configuration
  45 # VERIFY - run basic set of TCs with default settings
  46 TESTCASES_VERIFY="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
  47 TESTPARAM_VERIFY="--integration"
  48 TESTCASES_VERIFY_VPP="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_connection_vpp"
  49 TESTPARAM_VERIFY_VPP=$TESTPARAM_VERIFY
  50 # MERGE - run selected TCs with default settings
  51 TESTCASES_MERGE=$TESTCASES_VERIFY
  52 TESTPARAM_MERGE=$TESTPARAM_VERIFY
  53 TESTCASES_MERGE_VPP=$TESTCASES_VERIFY_VPP
  54 TESTPARAM_MERGE_VPP=$TESTPARAM_VERIFY_VPP
  55 # DAILY - run selected TCs for defined packet sizes
  56 TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back'
  57 TESTCASES_DAILY_VPP='phy2phy_tput_vpp phy2phy_back2back_vpp pvp_tput_vpp pvp_back2back_vpp pvvp_tput_vpp pvvp_back2back_vpp'
  58 TESTPARAM_DAILY='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
  59 TESTPARAM_DAILY_VPP=$TESTPARAM_DAILY
  60 TESTCASES_SRIOV='pvp_tput'
  61 TESTPARAM_SRIOV='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
  62 # check if user config file exists if not then we will use default settings
  63 if [ -f $HOME/vsperf-${BRANCH}.conf ] ; then
  64     # branch specific config was found
  65     CONF_FILE="--conf-file ${HOME}/vsperf-${BRANCH}.conf"
  66 else
  67     if [ -f $HOME/vsperf.conf ] ; then
  68         CONF_FILE="--conf-file ${HOME}/vsperf.conf"
  69     else
  70         CONF_FILE=""
  71     fi
  72 fi
  73 # check if sriov specific config file exists if not then use default configuration
  74 if [ -f $HOME/vsperf-${BRANCH}.conf.sriov ] ; then
  75     CONF_FILE_SRIOV="${CONF_FILE}.sriov"
  76 else
  77     CONF_FILE_SRIOV=$CONF_FILE
  78 fi
  79
  80 # Test report related configuration
  81 TEST_REPORT_PARTIAL="*_test_report.rst"
  82 TEST_REPORT_DIR="${WORKSPACE}/docs/results"
  83 TEST_REPORT_INDEX="${TEST_REPORT_DIR}/index.rst"
  84 TEST_REPORT_LINK_OLD="https://wiki.opnfv.org/wiki/vsperf_results"
  85 TEST_REPORT_FILE="${WORKSPACE}/docs_output/results/index.html"
  86 TEST_REPORT_TARBALL="vswitchperf_logs_${DATE}.tar.gz"
  87
  88 if [[ "x${BRANCH}" == "xmaster" ]]; then
  89     TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$DATE/${TEST_REPORT_TARBALL}"
  90 else
  91     TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$BRANCH/$DATE/${TEST_REPORT_TARBALL}"
  92 fi
  93
  94 TEST_REPORT_LOG_DIR="${HOME}/opnfv/$PROJECT/results/$BRANCH"
  95
  96 #
  97 # functions
  98 #
  99
 100 # terminate vsperf and all its utilities
 101 # it is expected that vsperf is the only python3 app
 102 # and no other ovs or qemu instances are running
 103 # at CI machine
 104 # parameters:
 105 #   none
 106 function terminate_vsperf() {
 107     sudo pkill stress &> /dev/null
 108     sudo pkill python3 &> /dev/null
 109     sudo killall -9 qemu-system-x86_64 &> /dev/null
 110
 111     # sometimes qemu resists to terminate, so wait a bit and kill it again
 112     if pgrep qemu-system-x86_64 &> /dev/null ; then
 113         sleep 5
 114         sudo killall -9 qemu-system-x86_64 &> /dev/null
 115         sleep 5
 116     fi
 117
 118     sudo pkill ovs-vswitchd &> /dev/null
 119     sleep 1
 120     sudo pkill ovsdb-server &> /dev/null
 121     sleep 1
 122 }
 123
 124 # check and print testcase execution status
 125 # parameters:
 126 #   $1 - directory with results
 127 function print_results() {
 128     for i in $TESTCASES ; do
 129         RES_FILE=`ls -1 $1 | egrep "result_${i}_[0-9a-zA-Z\-]+.csv"`
 130
 131         if [ "x$RES_FILE" != "x" -a -e "${1}/${RES_FILE}" ]; then
 132             if grep ^FAILED "${1}/${RES_FILE}" &> /dev/null ; then
 133                 printf "    %-70s %-6s\n" "result_${i}" "FAILED"
 134                 EXIT=$EXIT_TC_FAILED
 135             else
 136                 printf "    %-70s %-6s\n" "result_${i}" "OK"
 137             fi
 138         else
 139             printf "    %-70s %-6s\n" "result_${i}" "FAILED"
 140             EXIT=$EXIT_TC_FAILED
 141         fi
 142     done
 143 }
 144
 145 # execute tests and display results
 146 # parameters:
 147 #   $1 - vswitch and vnf combination, one of OVS_vanilla, OVS_with_DPDK_and_vHost_User
 148 #   $2 - CI job type, one of verify, merge, daily
 149 function execute_vsperf() {
 150     OPNFVPOD=""
 151     # figure out list of TCs and execution parameters
 152     case $2 in
 153         "verify")
 154             if [ "$1" == "VPP" ] ; then
 155                 TESTPARAM=$TESTPARAM_VERIFY_VPP
 156                 TESTCASES=$TESTCASES_VERIFY_VPP
 157             else
 158                 TESTPARAM=$TESTPARAM_VERIFY
 159                 TESTCASES=$TESTCASES_VERIFY
 160             fi
 161             ;;
 162         "merge")
 163             if [ "$1" == "VPP" ] ; then
 164                 TESTPARAM=$TESTPARAM_MERGE_VPP
 165                 TESTCASES=$TESTCASES_MERGE_VPP
 166             else
 167                 TESTPARAM=$TESTPARAM_MERGE
 168                 TESTCASES=$TESTCASES_MERGE
 169             fi
 170             ;;
 171         *)
 172             # by default use daily build and upload results to the OPNFV databse
 173             if [ "$1" == "VPP" ] ; then
 174                 TESTPARAM=$TESTPARAM_DAILY_VPP
 175                 TESTCASES=$TESTCASES_DAILY_VPP
 176                 # don't report VPP results into testresults DB, until TC name mapping
 177                 # for VPP tests will be defined
 178                 #OPNFVPOD="--opnfvpod=$NODE_NAME"
 179             else
 180                 TESTPARAM=$TESTPARAM_DAILY
 181                 TESTCASES=$TESTCASES_DAILY
 182                 OPNFVPOD="--opnfvpod=$NODE_NAME"
 183             fi
 184             ;;
 185     esac
 186
 187     # execute testcases
 188     echo -e "\nExecution of VSPERF for $1"
 189
 190     DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
 191
 192     case $1 in
 193         "SRIOV")
 194             # use SRIOV specific TCs and configuration
 195             TESTPARAM=$TESTPARAM_SRIOV
 196             TESTCASES=$TESTCASES_SRIOV
 197             # figure out log file name
 198             LOG_SUBDIR="SRIOV"
 199             LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
 200
 201             echo "    $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE"
 202             $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE
 203             ;;
 204         "VPP")
 205             # figure out log file name
 206             LOG_SUBDIR="VppDpdkVhost"
 207             LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
 208
 209             hugepages_info > $LOG_FILE
 210             echo "    $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
 211             $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
 212             hugepages_info >> $LOG_FILE
 213             ;;
 214         "OVS_vanilla")
 215             # figure out log file name
 216             LOG_SUBDIR="OvsVanilla"
 217             LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
 218
 219             echo "    $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE"
 220             $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
 221             ;;
 222         *)
 223             # figure out log file name
 224             LOG_SUBDIR="OvsDpdkVhost"
 225             LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
 226
 227             hugepages_info > $LOG_FILE
 228             echo "    $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
 229             $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
 230             hugepages_info >> $LOG_FILE
 231             ;;
 232     esac
 233
 234     # evaluation of results
 235     echo -e "\nResults for $1"
 236     RES_DIR="/$(grep "Creating result directory" $LOG_FILE | cut -d'/' -f2-)"
 237     if [[ "/" == "${RES_DIR}" ]] ; then
 238         echo "FAILURE: Results are not available."
 239         echo "-------------------------------------------------------------------"
 240         cat $LOG_FILE
 241         echo "-------------------------------------------------------------------"
 242         exit $EXIT_NO_RESULTS
 243     else
 244         print_results "${RES_DIR}"
 245         if [ $(($EXIT & $EXIT_TC_FAILED)) -gt 0 ] ; then
 246             echo "-------------------------------------------------------------------"
 247             cat $LOG_FILE
 248             echo "-------------------------------------------------------------------"
 249         fi
 250     fi
 251
 252     # show detailed result figures
 253     for md_file in $(grep '\.md"$' $LOG_FILE | cut -d'"' -f2); do
 254         # TC resut file header
 255         echo -e "\n-------------------------------------------------------------------"
 256         echo -e " $md_file"
 257         echo -e "-------------------------------------------------------------------\n"
 258         # TC details
 259         sed -n '/^- Test ID/,/Bidirectional/{/Packet size/b;p;/Bidirectional/q};/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file
 260         # TC results
 261         sed -n '/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file | grep -v "Unknown" | cat -s
 262     done
 263
 264     # add test results into the final doc template
 265     for report in ${RES_DIR}/${TEST_REPORT_PARTIAL} ; do
 266         # modify link to the artifactory with test report and logs
 267         if [ -f $report ] ; then
 268             sed -i -e "s,$TEST_REPORT_LINK_OLD,$TEST_REPORT_LINK_NEW," "$report"
 269             cp $report $TEST_REPORT_DIR
 270             echo "   $(basename $report)" >> $TEST_REPORT_INDEX
 271         fi
 272     done
 273
 274     # copy logs into dedicated directory
 275     mkdir ${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}
 276     [ -f "$LOG_FILE" ] && cp -a "${LOG_FILE}" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
 277     [ -d "$RES_DIR" ] && cp -ar "$RES_DIR" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
 278 }
 279
 280 # generates final test_report in PDF and HTML formats
 281 function generate_report() {
 282
 283     # prepare final tarball with all logs...
 284     tar --exclude "${TEST_REPORT_TARBALL}" -czf "${TEST_REPORT_LOG_DIR}/${TEST_REPORT_TARBALL}" $(find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d)
 285     # ...and remove original log files
 286     find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d -exec rm -rf \{\} \;
 287
 288     # clone opnfvdocs repository
 289     echo "Cloning opnfvdocs repository..."
 290     [ -d opnfvdocs ] && rm -rf opnfvdocs
 291     git clone https://gerrit.opnfv.org/gerrit/opnfvdocs &> /dev/null
 292
 293     # generate final docs with test results
 294     echo "Generating test report..."
 295     sed -ie 's,python ,python2 ,g' ./opnfvdocs/scripts/docs-build.sh
 296     OPNFVDOCS_DIR='./opnfvdocs' ./opnfvdocs/scripts/docs-build.sh &> /dev/null
 297
 298     # store HTML report with test results into dedicated directory
 299     if [ -f $TEST_REPORT_FILE ] ; then
 300         cp -ar $TEST_REPORT_FILE $(dirname $TEST_REPORT_FILE)/_static $TEST_REPORT_LOG_DIR
 301         echo "Final test report has been created."
 302     else
 303         echo "FAILURE: Generation of final test report has failed."
 304     fi
 305 }
 306
 307 # pushes test report and logs collected during test execution into artifactory
 308 function push_results_to_artifactory() {
 309     # clone releng repository
 310     echo "Cloning releng repository..."
 311     [ -d releng ] && rm -rf releng
 312     git clone https://gerrit.opnfv.org/gerrit/releng &> /dev/null
 313
 314     echo "Pushing results and logs into artifactory..."
 315     . ./releng/utils/push-test-logs.sh "$DATE"
 316
 317     # enter workspace as it could be modified by 3rd party script
 318     cd $WORKSPACE
 319 }
 320
 321 # removes any local changes of repository
 322 function cleanup() {
 323     echo "Cleaning up..."
 324     git stash -u
 325 }
 326
 327 # prepares directory for logs collection and removes old logs
 328 function initialize_logdir() {
 329     if [[ "x$TEST_REPORT_LOG_DIR" == "x" ]] ; then
 330         echo "FAILURE: Logging directory is not defined. Logs and report cannot be published!"
 331         exit $EXIT_NO_TEST_REPORT_LOG_DIR
 332     else
 333         # remove TEST_REPORT_LOG_DIR if it exists
 334         if [ -e $TEST_REPORT_LOG_DIR ] ; then
 335             if [ -f $TEST_REPORT_LOG_DIR ] ; then
 336                 rm $TEST_REPORT_LOG_DIR
 337             else
 338                 rm -rf ${TEST_REPORT_LOG_DIR}
 339             fi
 340         fi
 341         # create TEST_REPORT_LOG_DIR
 342         mkdir -p $TEST_REPORT_LOG_DIR
 343     fi
 344 }
 345
 346 # verify basic vsperf functionality
 347 function execute_vsperf_sanity() {
 348     DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
 349     LOG_FILE="${LOG_FILE_PREFIX}_sanity_${DATE_SUFFIX}.log"
 350     echo "Execution of VSPERF sanity checks:"
 351     for PARAM in '--version' '--help' '--list-trafficgens' '--list-collectors' '--list-vswitches' '--list-fwdapps' '--list-vnfs' '--list-settings' '--list' '--integration --list'; do
 352         echo -e "-------------------------------------------------------------------" >> $LOG_FILE
 353         echo "$VSPERF_BIN $PARAM $CONF_FILE" >> $LOG_FILE
 354         echo -e "-------------------------------------------------------------------" >> $LOG_FILE
 355         $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE
 356         if $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE ; then
 357             printf "    %-70s %-6s\n" "$VSPERF_BIN $PARAM" "OK"
 358         else
 359             printf "    %-70s %-6s\n" "$VSPERF_BIN $PARAM" "FAILED"
 360             EXIT=$EXIT_SANITY_TC_FAILED
 361         fi
 362         echo >> $LOG_FILE
 363     done
 364     echo "Sanity log file $LOG_FILE"
 365     if [ $(($EXIT & $EXIT_SANITY_TC_FAILED)) -gt 0 ] ; then
 366         echo "-------------------------------------------------------------------"
 367         cat $LOG_FILE
 368         echo "-------------------------------------------------------------------"
 369     fi
 370 }
 371
 372 # execute pylint to check code quality
 373 function execute_vsperf_pylint_check() {
 374     if ! ./check -b ; then
 375         EXIT=$EXIT_PYLINT_FAILED
 376     fi
 377 }
 378
 379 # check and install required packages at nodes running VERIFY and MERGE jobs
 380 function dependencies_check() {
 381     . /etc/os-release
 382     if [ $ID == "ubuntu" ] ; then
 383         echo "Dependencies check"
 384         echo "=================="
 385         # install system packages
 386         for PACKAGE in "python3-tk" "sysstat" "bc" ; do
 387             if dpkg -s $PACKAGE &> /dev/null ; then
 388                 printf "    %-70s %-6s\n" $PACKAGE "OK"
 389             else
 390                 printf "    %-70s %-6s\n" $PACKAGE "missing"
 391                 sudo apt-get install -y $PACKAGE
 392             fi
 393         done
 394         # install additional python packages into python environment
 395         for PACKAGE in "pylint" ; do
 396             if pip show $PACKAGE &> /dev/null ; then
 397                 printf "    %-70s %-6s\n" $PACKAGE "OK"
 398             else
 399                 printf "    %-70s %-6s\n" $PACKAGE "missing"
 400                 pip install $PACKAGE
 401             fi
 402         done
 403         echo
 404     fi
 405 }
 406
 407 # configure hugepages
 408 function configure_hugepages() {
 409     sudo bash -c "echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
 410     sudo bash -c "echo 0 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
 411 }
 412
 413 # dump hugepages configuration
 414 function hugepages_info() {
 415     echo "-------------------------------------------------------------------"
 416     head /sys/devices/system/node/node*/hugepages/hugepages*/*
 417     echo "-------------------------------------------------------------------"
 418 }
 419
 420 #
 421 # main
 422 #
 423
 424 echo
 425
 426 # enter workspace dir
 427 cd $WORKSPACE
 428
 429 # create virtualenv if needed
 430 if [ ! -e $VSPERFENV_DIR ] ; then
 431     echo "Create VSPERF environment"
 432     echo "========================="
 433     virtualenv --python=python3 "$VSPERFENV_DIR"
 434     echo
 435 fi
 436
 437 # acivate and update virtualenv
 438 echo "Update VSPERF environment"
 439 echo "========================="
 440 source "$VSPERFENV_DIR"/bin/activate
 441 pip install -r ./requirements.txt
 442 echo
 443
 444 # VERFIY&MERGE job specific - check if required packages are installed
 445 dependencies_check
 446
 447 # initialization
 448 initialize_logdir
 449
 450 # configure hugepages
 451 configure_hugepages
 452
 453 # execute job based on passed parameter
 454 case $1 in
 455     "verify")
 456         echo "================="
 457         echo "VSPERF verify job"
 458         echo "================="
 459
 460         execute_vsperf_pylint_check
 461         terminate_vsperf
 462         execute_vsperf_sanity
 463         terminate_vsperf
 464         execute_vsperf OVS_with_DPDK_and_vHost_User $1
 465         terminate_vsperf
 466         execute_vsperf OVS_vanilla $1
 467         terminate_vsperf
 468         execute_vsperf VPP $1
 469         terminate_vsperf
 470
 471         exit $EXIT
 472         ;;
 473     "merge")
 474         echo "================"
 475         echo "VSPERF merge job"
 476         echo "================"
 477
 478         execute_pylint_check
 479         terminate_vsperf
 480         execute_vsperf_sanity
 481         terminate_vsperf
 482         execute_vsperf OVS_with_DPDK_and_vHost_User $1
 483         terminate_vsperf
 484         execute_vsperf OVS_vanilla $1
 485         terminate_vsperf
 486         execute_vsperf VPP $1
 487         terminate_vsperf
 488
 489         exit $EXIT
 490         ;;
 491     *)
 492         echo "================"
 493         echo "VSPERF daily job"
 494         echo "================"
 495
 496         terminate_vsperf
 497         execute_vsperf OVS_with_DPDK_and_vHost_User $1
 498         terminate_vsperf
 499         execute_vsperf OVS_vanilla $1
 500         terminate_vsperf
 501         execute_vsperf VPP $1
 502         terminate_vsperf
 503         execute_vsperf SRIOV $1
 504         terminate_vsperf
 505
 506         generate_report
 507
 508         push_results_to_artifactory
 509
 510         cleanup
 511
 512         exit $EXIT
 513         ;;
 514 esac
 515
 516 exit $EXIT
 517
 518 #
 519 # end
 520 #