3 # Copyright 2015-2017 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # VSPERF nightly build execution script
20 # build-vsperf.sh job_type
21 # where job_type is one of "verify", "merge", "daily"
29 EXIT_SANITY_TC_FAILED=2
32 EXIT_NO_TEST_REPORT_LOG_DIR=256
39 LOG_FILE_PREFIX="/tmp/vsperf_build"
40 DATE=$(date -u +"%Y-%m-%d_%H-%M-%S")
41 BRANCH=${GIT_BRANCH##*/}
42 VSPERFENV_DIR="$HOME/vsperfenv"
43 RESULTS_ARCHIVE="$HOME/ci_results_archive"
45 # CI job specific configuration
46 # VERIFY - run basic set of TCs with default settings
47 TESTCASES_VERIFY="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
48 TESTPARAM_VERIFY="--integration"
49 TESTCASES_VERIFY_VPP="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_connection_vpp"
50 TESTPARAM_VERIFY_VPP=$TESTPARAM_VERIFY
51 # MERGE - run selected TCs with default settings
52 TESTCASES_MERGE=$TESTCASES_VERIFY
53 TESTPARAM_MERGE=$TESTPARAM_VERIFY
54 TESTCASES_MERGE_VPP=$TESTCASES_VERIFY_VPP
55 TESTPARAM_MERGE_VPP=$TESTPARAM_VERIFY_VPP
56 # DAILY - run selected TCs for defined packet sizes
57 TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back'
58 TESTCASES_DAILY_VPP='phy2phy_tput_vpp phy2phy_back2back_vpp pvp_tput_vpp pvp_back2back_vpp pvvp_tput_vpp pvvp_back2back_vpp'
59 TESTPARAM_DAILY='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
60 TESTPARAM_DAILY_VPP=$TESTPARAM_DAILY
61 TESTCASES_SRIOV='pvp_tput'
62 TESTPARAM_SRIOV='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
63 # check if user config file exists if not then we will use default settings
64 if [ -f $HOME/vsperf-${BRANCH}.conf ] ; then
65 # branch specific config was found
66 CONF_FILE="--conf-file ${HOME}/vsperf-${BRANCH}.conf"
68 if [ -f $HOME/vsperf.conf ] ; then
69 CONF_FILE="--conf-file ${HOME}/vsperf.conf"
74 # check if sriov specific config file exists if not then use default configuration
75 if [ -f $HOME/vsperf-${BRANCH}.conf.sriov ] ; then
76 CONF_FILE_SRIOV="${CONF_FILE}.sriov"
78 CONF_FILE_SRIOV=$CONF_FILE
81 # Test report related configuration
82 TEST_REPORT_PARTIAL="*_test_report.rst"
83 TEST_REPORT_DIR="${WORKSPACE}/docs/testing/developer/devguide/results"
84 TEST_REPORT_INDEX="${TEST_REPORT_DIR}/index.rst"
85 TEST_REPORT_LINK_OLD="https://wiki.opnfv.org/wiki/vsperf_results"
86 TEST_REPORT_FILE="${WORKSPACE}/docs_output/testing_developer_devguide_results/index.html"
87 TEST_REPORT_TARBALL="vswitchperf_logs_${DATE}.tar.gz"
89 if [[ "x${BRANCH}" == "xmaster" ]]; then
90 TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$DATE/${TEST_REPORT_TARBALL}"
92 TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$BRANCH/$DATE/${TEST_REPORT_TARBALL}"
95 TEST_REPORT_LOG_DIR="${HOME}/opnfv/$PROJECT/results/$BRANCH"
101 # terminate vsperf and all its utilities
102 # it is expected that vsperf is the only python3 app
103 # and no other ovs or qemu instances are running
107 function terminate_vsperf() {
108 sudo pkill stress &> /dev/null
109 sudo pkill python3 &> /dev/null
110 sudo killall -9 qemu-system-x86_64 &> /dev/null
112 # sometimes qemu resists to terminate, so wait a bit and kill it again
113 if pgrep qemu-system-x86_64 &> /dev/null ; then
115 sudo killall -9 qemu-system-x86_64 &> /dev/null
119 sudo pkill ovs-vswitchd &> /dev/null
121 sudo pkill ovsdb-server &> /dev/null
123 sudo pkill vppctl &> /dev/null
125 sudo pkill vpp &> /dev/null
127 sudo pkill -9 vpp &> /dev/null
131 # check and print testcase execution status
133 # $1 - directory with results
134 function print_results() {
135 for i in $TESTCASES ; do
136 if [ ! -e $1 ] ; then
137 printf " %-70s %-6s\n" "result_${i}" "FAILED"
140 RES_FILE=`ls -1 $1 | egrep "result_${i}_[0-9a-zA-Z\-]+.csv"`
142 if [ "x$RES_FILE" != "x" -a -e "${1}/${RES_FILE}" ]; then
143 if grep ^FAILED "${1}/${RES_FILE}" &> /dev/null ; then
144 printf " %-70s %-6s\n" "result_${i}" "FAILED"
147 printf " %-70s %-6s\n" "result_${i}" "OK"
150 printf " %-70s %-6s\n" "result_${i}" "FAILED"
157 # execute tests and display results
159 # $1 - vswitch and vnf combination, one of OVS_vanilla, OVS_with_DPDK_and_vHost_User
160 # $2 - CI job type, one of verify, merge, daily
161 function execute_vsperf() {
163 # figure out list of TCs and execution parameters
166 if [ "$1" == "VPP" ] ; then
167 TESTPARAM=$TESTPARAM_VERIFY_VPP
168 TESTCASES=$TESTCASES_VERIFY_VPP
170 TESTPARAM=$TESTPARAM_VERIFY
171 TESTCASES=$TESTCASES_VERIFY
175 if [ "$1" == "VPP" ] ; then
176 TESTPARAM=$TESTPARAM_MERGE_VPP
177 TESTCASES=$TESTCASES_MERGE_VPP
179 TESTPARAM=$TESTPARAM_MERGE
180 TESTCASES=$TESTCASES_MERGE
184 # by default use daily build and upload results to the OPNFV databse
185 if [ "$1" == "VPP" ] ; then
186 TESTPARAM=$TESTPARAM_DAILY_VPP
187 TESTCASES=$TESTCASES_DAILY_VPP
188 # don't report VPP results into testresults DB, until TC name mapping
189 # for VPP tests will be defined
190 #OPNFVPOD="--opnfvpod=$NODE_NAME"
192 TESTPARAM=$TESTPARAM_DAILY
193 TESTCASES=$TESTCASES_DAILY
194 OPNFVPOD="--opnfvpod=$NODE_NAME"
200 echo -e "\nExecution of VSPERF for $1"
202 DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
206 # use SRIOV specific TCs and configuration
207 TESTPARAM=$TESTPARAM_SRIOV
208 TESTCASES=$TESTCASES_SRIOV
209 # figure out log file name
211 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
213 echo " $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE"
214 $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE
217 # figure out log file name
218 LOG_SUBDIR="VppDpdkVhost"
219 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
221 hugepages_info > $LOG_FILE
223 # check if VPP is up & running
224 echo " $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM vswitch_version_vpp >> ${LOG_FILE}2"
225 $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM vswitch_version_vpp &>> ${LOG_FILE}2
226 if (grep FAILED ${LOG_FILE}2 &> /dev/null ) ; then
227 # VPP can't be executed or vppctl can't connect to it, so skip
228 # VPP tests as it is probably an issue with jenkins slave
229 printf " %-70s %-6s\n" "vswitch_version_vpp" "FAILED"
231 echo "VPP is not working properly. VPP tests will be skipped."
235 printf " %-70s %-6s\n" "vswitch_version_vpp" "OK"
237 echo " $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
238 $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
239 hugepages_info >> $LOG_FILE
242 # figure out log file name
243 LOG_SUBDIR="OvsVanilla"
244 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
246 echo " $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE"
247 $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
250 # figure out log file name
251 LOG_SUBDIR="OvsDpdkVhost"
252 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
254 hugepages_info > $LOG_FILE
255 echo " $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
256 $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
257 hugepages_info >> $LOG_FILE
261 # evaluation of results
262 echo -e "\nResults for $1"
263 RES_DIR="/$(grep "Creating result directory" $LOG_FILE | cut -d'/' -f2-)"
264 if [[ "/" == "${RES_DIR}" ]] ; then
265 echo "FAILURE: Results are not available."
266 echo "-------------------------------------------------------------------"
268 echo "-------------------------------------------------------------------"
269 exit $EXIT_NO_RESULTS
271 print_results "${RES_DIR}"
272 if [ $(($EXIT & $EXIT_TC_FAILED)) -gt 0 ] ; then
273 echo "-------------------------------------------------------------------"
275 echo "-------------------------------------------------------------------"
279 # show detailed result figures
280 for md_file in $(grep '\.md"$' $LOG_FILE | cut -d'"' -f2); do
281 # TC resut file header
282 echo -e "\n-------------------------------------------------------------------"
284 echo -e "-------------------------------------------------------------------\n"
286 sed -n '/^- Test ID/,/Bidirectional/{/Packet size/b;p;/Bidirectional/q};/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file
288 sed -n '/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file | grep -v "Unknown" | cat -s
291 # add test results into the final doc template
292 for report in ${RES_DIR}/${TEST_REPORT_PARTIAL} ; do
293 # modify link to the artifactory with test report and logs
294 if [ -f $report ] ; then
295 sed -i -e "s,$TEST_REPORT_LINK_OLD,$TEST_REPORT_LINK_NEW," "$report"
296 cp $report $TEST_REPORT_DIR
297 echo " $(basename $report)" >> $TEST_REPORT_INDEX
301 # copy logs into dedicated directory
302 mkdir ${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}
303 [ -f "$LOG_FILE" ] && cp -a "${LOG_FILE}" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
304 [ -d "$RES_DIR" ] && cp -ar "$RES_DIR" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
307 # generates final test_report in PDF and HTML formats
308 function generate_report() {
310 # prepare final tarball with all logs...
311 tar --exclude "${TEST_REPORT_TARBALL}" -czf "${TEST_REPORT_LOG_DIR}/${TEST_REPORT_TARBALL}" $(find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d)
312 # ...and move original log files to the archive directory...
313 find "${TEST_REPORT_LOG_DIR}" -maxdepth 2 -name "results_*" -type d -exec mv \{\} ${RESULTS_ARCHIVE} \;
314 # ...and remove the rest
315 find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d -exec rm -rf \{\} \;
317 # clone opnfvdocs repository
318 echo "Cloning opnfvdocs repository..."
319 [ -d opnfvdocs ] && rm -rf opnfvdocs
320 git clone https://gerrit.opnfv.org/gerrit/opnfvdocs &> /dev/null
322 # generate final docs with test results
323 echo "Generating test report..."
324 sed -ie 's,python ,python2 ,g' ./opnfvdocs/scripts/docs-build.sh
325 OPNFVDOCS_DIR='./opnfvdocs' ./opnfvdocs/scripts/docs-build.sh &> /dev/null
327 # store HTML report with test results into dedicated directory
328 if [ -f $TEST_REPORT_FILE ] ; then
329 cp -ar $TEST_REPORT_FILE $(dirname $TEST_REPORT_FILE)/_static $TEST_REPORT_LOG_DIR
330 echo "Final test report has been created."
332 echo "FAILURE: Generation of final test report has failed."
336 # generates graphs from recent test results
337 function generate_and_push_graphs() {
338 # create graphs from results in archive directory
339 ./ci/plot-results.sh "$1" "$2" "$RESULTS_ARCHIVE"
341 # push graphs into artifactory
342 if ls *png &> /dev/null ; then
343 gsutil cp *png gs://artifacts.opnfv.org/logs/vswitchperf/intel-pod12/graphs/
345 echo "Graphs were not created."
349 # pushes test report and logs collected during test execution into artifactory
350 function push_results_to_artifactory() {
351 # clone releng repository
352 echo "Cloning releng repository..."
353 [ -d releng ] && rm -rf releng
354 git clone https://gerrit.opnfv.org/gerrit/releng &> /dev/null
356 echo "Pushing results and logs into artifactory..."
357 . ./releng/utils/push-test-logs.sh "$DATE"
359 # enter workspace as it could be modified by 3rd party script
363 # removes any local changes of repository
365 echo "Cleaning up..."
369 # prepares directory for logs collection and removes old logs
370 function initialize_logdir() {
371 if [[ "x$TEST_REPORT_LOG_DIR" == "x" ]] ; then
372 echo "FAILURE: Logging directory is not defined. Logs and report cannot be published!"
373 exit $EXIT_NO_TEST_REPORT_LOG_DIR
375 # remove TEST_REPORT_LOG_DIR if it exists
376 if [ -e $TEST_REPORT_LOG_DIR ] ; then
377 if [ -f $TEST_REPORT_LOG_DIR ] ; then
378 rm $TEST_REPORT_LOG_DIR
380 rm -rf ${TEST_REPORT_LOG_DIR}
383 # create TEST_REPORT_LOG_DIR
384 mkdir -p $TEST_REPORT_LOG_DIR
388 # verify basic vsperf functionality
389 function execute_vsperf_sanity() {
390 DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
391 LOG_FILE="${LOG_FILE_PREFIX}_sanity_${DATE_SUFFIX}.log"
392 echo "Execution of VSPERF sanity checks:"
393 for PARAM in '--version' '--help' '--list-trafficgens' '--list-collectors' '--list-vswitches' '--list-fwdapps' '--list-vnfs' '--list-loadgens' '--list-settings' '--list' '--integration --list'; do
394 echo -e "-------------------------------------------------------------------" >> $LOG_FILE
395 echo "$VSPERF_BIN $PARAM $CONF_FILE" >> $LOG_FILE
396 echo -e "-------------------------------------------------------------------" >> $LOG_FILE
397 $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE
398 if $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE ; then
399 printf " %-70s %-6s\n" "$VSPERF_BIN $PARAM" "OK"
401 printf " %-70s %-6s\n" "$VSPERF_BIN $PARAM" "FAILED"
402 EXIT=$EXIT_SANITY_TC_FAILED
406 echo "Sanity log file $LOG_FILE"
407 if [ $(($EXIT & $EXIT_SANITY_TC_FAILED)) -gt 0 ] ; then
408 echo "-------------------------------------------------------------------"
410 echo "-------------------------------------------------------------------"
414 # execute pylint to check code quality
415 function execute_vsperf_pylint_check() {
416 if ! ./check -b ; then
417 EXIT=$EXIT_PYLINT_FAILED
421 # check and install required packages at nodes running VERIFY and MERGE jobs
422 function dependencies_check() {
424 if [ $ID == "ubuntu" ] ; then
425 echo "Dependencies check"
426 echo "=================="
427 # install system packages
428 for PACKAGE in "python3-tk" "sysstat" "bc" ; do
429 if dpkg -s $PACKAGE &> /dev/null ; then
430 printf " %-70s %-6s\n" $PACKAGE "OK"
432 printf " %-70s %-6s\n" $PACKAGE "missing"
433 sudo apt-get install -y $PACKAGE
436 # install additional python packages into python environment
437 for PACKAGE in "pylint" ; do
438 if pip show $PACKAGE &> /dev/null ; then
439 printf " %-70s %-6s\n" $PACKAGE "OK"
441 printf " %-70s %-6s\n" $PACKAGE "missing"
449 # configure hugepages
450 function configure_hugepages() {
453 HP_NR=`cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages`
454 HP_FREE=`cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/free_hugepages`
455 # check if HP must be (re)configured
456 if [ $HP_FREE -lt $HP_REQUESTED ] ; then
457 HP_NR_NEW=$(($HP_NR+($HP_REQUESTED-$HP_FREE)))
458 if [ $HP_NR_NEW -gt $HP_MAX ] ; then
461 sudo bash -c "echo $HP_NR_NEW > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
464 if [ -f /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages ] ; then
465 sudo bash -c "echo 0 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
469 # dump hugepages configuration
470 function hugepages_info() {
471 echo "-------------------------------------------------------------------"
472 head /sys/devices/system/node/node*/hugepages/hugepages*/*
473 echo "-------------------------------------------------------------------"
482 # enter workspace dir
485 # create virtualenv if needed
486 if [ ! -e $VSPERFENV_DIR ] ; then
487 echo "Create VSPERF environment"
488 echo "========================="
489 virtualenv --python=python3 "$VSPERFENV_DIR"
493 # acivate and update virtualenv
494 echo "Update VSPERF environment"
495 echo "========================="
496 source "$VSPERFENV_DIR"/bin/activate
497 pip install -r ./requirements.txt
500 # VERFIY&MERGE job specific - check if required packages are installed
506 # configure hugepages
507 echo "Configure hugepages"
508 echo "==================="
510 hugepages_info | grep -v '^--'
513 # execute job based on passed parameter
516 echo "================="
517 echo "VSPERF verify job"
518 echo "================="
520 execute_vsperf_pylint_check
522 execute_vsperf_sanity
524 execute_vsperf OVS_with_DPDK_and_vHost_User $1
526 execute_vsperf OVS_vanilla $1
528 execute_vsperf VPP $1
534 echo "================"
535 echo "VSPERF merge job"
536 echo "================"
540 execute_vsperf_sanity
542 execute_vsperf OVS_with_DPDK_and_vHost_User $1
544 execute_vsperf OVS_vanilla $1
546 execute_vsperf VPP $1
552 echo "================"
553 echo "VSPERF daily job"
554 echo "================"
557 execute_vsperf OVS_with_DPDK_and_vHost_User $1
559 execute_vsperf OVS_vanilla $1
561 execute_vsperf VPP $1
563 execute_vsperf SRIOV $1
568 push_results_to_artifactory
570 generate_and_push_graphs "$TESTCASES_DAILY" ",OvsDpdkVhost,"
571 generate_and_push_graphs "$TESTCASES_DAILY" ",OvsVanilla,"
572 generate_and_push_graphs "$TESTCASES_DAILY_VPP" ",VppDpdkVhost,"
573 generate_and_push_graphs "$TESTCASES_SRIOV" ",none,"