3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # VSPERF nightly build execution script
20 # build-vsperf.sh job_type
21 # where job_type is one of "verify", "merge", "daily"
29 EXIT_SANITY_TC_FAILED=2
32 EXIT_NO_TEST_REPORT_LOG_DIR=256
39 LOG_FILE_PREFIX="/tmp/vsperf_build"
40 DATE=$(date -u +"%Y-%m-%d_%H-%M-%S")
41 BRANCH=${GIT_BRANCH##*/}
42 VSPERFENV_DIR="$HOME/vsperfenv"
44 # CI job specific configuration
45 # VERIFY - run basic set of TCs with default settings
46 TESTCASES_VERIFY="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
47 TESTPARAM_VERIFY="--integration"
48 TESTCASES_VERIFY_VPP="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_connection_vpp"
49 TESTPARAM_VERIFY_VPP=$TESTPARAM_VERIFY
50 # MERGE - run selected TCs with default settings
51 TESTCASES_MERGE=$TESTCASES_VERIFY
52 TESTPARAM_MERGE=$TESTPARAM_VERIFY
53 TESTCASES_MERGE_VPP=$TESTCASES_VERIFY_VPP
54 TESTPARAM_MERGE_VPP=$TESTPARAM_VERIFY_VPP
55 # DAILY - run selected TCs for defined packet sizes
56 TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back'
57 TESTCASES_DAILY_VPP='phy2phy_tput_vpp back2back_vpp pvp_tput_vpp pvp_back2back_vpp pvvp_tput_vpp pvvp_back2back_vpp'
58 TESTPARAM_DAILY='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
59 TESTPARAM_DAILY_VPP=$TESTPARAM_DAILY
60 TESTCASES_SRIOV='pvp_tput'
61 TESTPARAM_SRIOV='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
62 # check if user config file exists if not then we will use default settings
63 if [ -f $HOME/vsperf-${BRANCH}.conf ] ; then
64 # branch specific config was found
65 CONF_FILE="--conf-file ${HOME}/vsperf-${BRANCH}.conf"
67 if [ -f $HOME/vsperf.conf ] ; then
68 CONF_FILE="--conf-file ${HOME}/vsperf.conf"
73 # check if sriov specific config file exists if not then use default configuration
74 if [ -f $HOME/vsperf-${BRANCH}.conf.sriov ] ; then
75 CONF_FILE_SRIOV="${CONF_FILE}.sriov"
77 CONF_FILE_SRIOV=$CONF_FILE
80 # Test report related configuration
81 TEST_REPORT_PARTIAL="*_test_report.rst"
82 TEST_REPORT_DIR="${WORKSPACE}/docs/results"
83 TEST_REPORT_INDEX="${TEST_REPORT_DIR}/index.rst"
84 TEST_REPORT_LINK_OLD="https://wiki.opnfv.org/wiki/vsperf_results"
85 TEST_REPORT_FILE="${WORKSPACE}/docs_output/results/index.html"
86 TEST_REPORT_TARBALL="vswitchperf_logs_${DATE}.tar.gz"
88 if [[ "x${BRANCH}" == "xmaster" ]]; then
89 TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$DATE/${TEST_REPORT_TARBALL}"
91 TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$BRANCH/$DATE/${TEST_REPORT_TARBALL}"
94 TEST_REPORT_LOG_DIR="${HOME}/opnfv/$PROJECT/results/$BRANCH"
100 # terminate vsperf and all its utilities
101 # it is expected that vsperf is the only python3 app
102 # and no other ovs or qemu instances are running
106 function terminate_vsperf() {
107 sudo pkill stress &> /dev/null
108 sudo pkill python3 &> /dev/null
109 sudo killall -9 qemu-system-x86_64 &> /dev/null
111 # sometimes qemu resists to terminate, so wait a bit and kill it again
112 if pgrep qemu-system-x86_64 &> /dev/null ; then
114 sudo killall -9 qemu-system-x86_64 &> /dev/null
118 sudo pkill ovs-vswitchd &> /dev/null
120 sudo pkill ovsdb-server &> /dev/null
124 # check and print testcase execution status
126 # $1 - directory with results
127 function print_results() {
128 for i in $TESTCASES ; do
129 RES_FILE=`ls -1 $1 | egrep "result_${i}_[0-9a-zA-Z\-]+.csv"`
131 if [ "x$RES_FILE" != "x" -a -e "${1}/${RES_FILE}" ]; then
132 if grep ^FAILED "${1}/${RES_FILE}" &> /dev/null ; then
133 printf " %-70s %-6s\n" "result_${i}" "FAILED"
136 printf " %-70s %-6s\n" "result_${i}" "OK"
139 printf " %-70s %-6s\n" "result_${i}" "FAILED"
145 # execute tests and display results
147 # $1 - vswitch and vnf combination, one of OVS_vanilla, OVS_with_DPDK_and_vHost_User
148 # $2 - CI job type, one of verify, merge, daily
149 function execute_vsperf() {
151 # figure out list of TCs and execution parameters
154 if [ "$1" == "VPP" ] ; then
155 TESTPARAM=$TESTPARAM_VERIFY_VPP
156 TESTCASES=$TESTCASES_VERIFY_VPP
158 TESTPARAM=$TESTPARAM_VERIFY
159 TESTCASES=$TESTCASES_VERIFY
163 if [ "$1" == "VPP" ] ; then
164 TESTPARAM=$TESTPARAM_MERGE_VPP
165 TESTCASES=$TESTCASES_MERGE_VPP
167 TESTPARAM=$TESTPARAM_MERGE
168 TESTCASES=$TESTCASES_MERGE
172 # by default use daily build and upload results to the OPNFV databse
173 if [ "$1" == "VPP" ] ; then
174 TESTPARAM=$TESTPARAM_DAILY_VPP
175 TESTCASES=$TESTCASES_DAILY_VPP
176 # don't report VPP results into testresults DB, until TC name mapping
177 # for VPP tests will be defined
178 #OPNFVPOD="--opnfvpod=$NODE_NAME"
180 TESTPARAM=$TESTPARAM_DAILY
181 TESTCASES=$TESTCASES_DAILY
182 OPNFVPOD="--opnfvpod=$NODE_NAME"
188 echo -e "\nExecution of VSPERF for $1"
190 DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
194 # use SRIOV specific TCs and configuration
195 TESTPARAM=$TESTPARAM_SRIOV
196 TESTCASES=$TESTCASES_SRIOV
197 # figure out log file name
199 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
201 echo " $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE"
202 $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE
205 # figure out log file name
206 LOG_SUBDIR="VppDpdkVhost"
207 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
209 hugepages_info > $LOG_FILE
210 echo " $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
211 $VSPERF_BIN $OPNFVPOD --vswitch VppDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
212 hugepages_info >> $LOG_FILE
215 # figure out log file name
216 LOG_SUBDIR="OvsVanilla"
217 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
219 echo " $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE"
220 $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
223 # figure out log file name
224 LOG_SUBDIR="OvsDpdkVhost"
225 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
227 hugepages_info > $LOG_FILE
228 echo " $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
229 $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
230 hugepages_info >> $LOG_FILE
234 # evaluation of results
235 echo -e "\nResults for $1"
236 RES_DIR="/$(grep "Creating result directory" $LOG_FILE | cut -d'/' -f2-)"
237 if [[ "/" == "${RES_DIR}" ]] ; then
238 echo "FAILURE: Results are not available."
239 echo "-------------------------------------------------------------------"
241 echo "-------------------------------------------------------------------"
242 exit $EXIT_NO_RESULTS
244 print_results "${RES_DIR}"
245 if [ $(($EXIT & $EXIT_TC_FAILED)) -gt 0 ] ; then
246 echo "-------------------------------------------------------------------"
248 echo "-------------------------------------------------------------------"
252 # show detailed result figures
253 for md_file in $(grep '\.md"$' $LOG_FILE | cut -d'"' -f2); do
254 # TC resut file header
255 echo -e "\n-------------------------------------------------------------------"
257 echo -e "-------------------------------------------------------------------\n"
259 sed -n '/^- Test ID/,/Bidirectional/{/Packet size/b;p;/Bidirectional/q};/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file
261 sed -n '/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file | grep -v "Unknown" | cat -s
264 # add test results into the final doc template
265 for report in ${RES_DIR}/${TEST_REPORT_PARTIAL} ; do
266 # modify link to the artifactory with test report and logs
267 if [ -f $report ] ; then
268 sed -i -e "s,$TEST_REPORT_LINK_OLD,$TEST_REPORT_LINK_NEW," "$report"
269 cp $report $TEST_REPORT_DIR
270 echo " $(basename $report)" >> $TEST_REPORT_INDEX
274 # copy logs into dedicated directory
275 mkdir ${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}
276 [ -f "$LOG_FILE" ] && cp -a "${LOG_FILE}" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
277 [ -d "$RES_DIR" ] && cp -ar "$RES_DIR" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
280 # generates final test_report in PDF and HTML formats
281 function generate_report() {
283 # prepare final tarball with all logs...
284 tar --exclude "${TEST_REPORT_TARBALL}" -czf "${TEST_REPORT_LOG_DIR}/${TEST_REPORT_TARBALL}" $(find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d)
285 # ...and remove original log files
286 find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d -exec rm -rf \{\} \;
288 # clone opnfvdocs repository
289 echo "Cloning opnfvdocs repository..."
290 [ -d opnfvdocs ] && rm -rf opnfvdocs
291 git clone https://gerrit.opnfv.org/gerrit/opnfvdocs &> /dev/null
293 # generate final docs with test results
294 echo "Generating test report..."
295 sed -ie 's,python ,python2 ,g' ./opnfvdocs/scripts/docs-build.sh
296 OPNFVDOCS_DIR='./opnfvdocs' ./opnfvdocs/scripts/docs-build.sh &> /dev/null
298 # store HTML report with test results into dedicated directory
299 if [ -f $TEST_REPORT_FILE ] ; then
300 cp -ar $TEST_REPORT_FILE $(dirname $TEST_REPORT_FILE)/_static $TEST_REPORT_LOG_DIR
301 echo "Final test report has been created."
303 echo "FAILURE: Generation of final test report has failed."
307 # pushes test report and logs collected during test execution into artifactory
308 function push_results_to_artifactory() {
309 # clone releng repository
310 echo "Cloning releng repository..."
311 [ -d releng ] && rm -rf releng
312 git clone https://gerrit.opnfv.org/gerrit/releng &> /dev/null
314 echo "Pushing results and logs into artifactory..."
315 . ./releng/utils/push-test-logs.sh "$DATE"
317 # enter workspace as it could be modified by 3rd party script
321 # removes any local changes of repository
323 echo "Cleaning up..."
327 # prepares directory for logs collection and removes old logs
328 function initialize_logdir() {
329 if [[ "x$TEST_REPORT_LOG_DIR" == "x" ]] ; then
330 echo "FAILURE: Logging directory is not defined. Logs and report cannot be published!"
331 exit $EXIT_NO_TEST_REPORT_LOG_DIR
333 # remove TEST_REPORT_LOG_DIR if it exists
334 if [ -e $TEST_REPORT_LOG_DIR ] ; then
335 if [ -f $TEST_REPORT_LOG_DIR ] ; then
336 rm $TEST_REPORT_LOG_DIR
338 rm -rf ${TEST_REPORT_LOG_DIR}
341 # create TEST_REPORT_LOG_DIR
342 mkdir -p $TEST_REPORT_LOG_DIR
346 # verify basic vsperf functionality
347 function execute_vsperf_sanity() {
348 DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
349 LOG_FILE="${LOG_FILE_PREFIX}_sanity_${DATE_SUFFIX}.log"
350 echo "Execution of VSPERF sanity checks:"
351 for PARAM in '--version' '--help' '--list-trafficgens' '--list-collectors' '--list-vswitches' '--list-fwdapps' '--list-vnfs' '--list-settings' '--list' '--integration --list'; do
352 echo -e "-------------------------------------------------------------------" >> $LOG_FILE
353 echo "$VSPERF_BIN $PARAM $CONF_FILE" >> $LOG_FILE
354 echo -e "-------------------------------------------------------------------" >> $LOG_FILE
355 $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE
356 if $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE ; then
357 printf " %-70s %-6s\n" "$VSPERF_BIN $PARAM" "OK"
359 printf " %-70s %-6s\n" "$VSPERF_BIN $PARAM" "FAILED"
360 EXIT=$EXIT_SANITY_TC_FAILED
364 echo "Sanity log file $LOG_FILE"
365 if [ $(($EXIT & $EXIT_SANITY_TC_FAILED)) -gt 0 ] ; then
366 echo "-------------------------------------------------------------------"
368 echo "-------------------------------------------------------------------"
372 # execute pylint to check code quality
373 function execute_vsperf_pylint_check() {
374 if ! ./check -b ; then
375 EXIT=$EXIT_PYLINT_FAILED
379 # check and install required packages at nodes running VERIFY and MERGE jobs
380 function dependencies_check() {
382 if [ $ID == "ubuntu" ] ; then
383 echo "Dependencies check"
384 echo "=================="
385 # install system packages
386 for PACKAGE in "python3-tk" "sysstat" "bc" ; do
387 if dpkg -s $PACKAGE &> /dev/null ; then
388 printf " %-70s %-6s\n" $PACKAGE "OK"
390 printf " %-70s %-6s\n" $PACKAGE "missing"
391 sudo apt-get install -y $PACKAGE
394 # install additional python packages into python environment
395 for PACKAGE in "pylint" ; do
396 if pip show $PACKAGE &> /dev/null ; then
397 printf " %-70s %-6s\n" $PACKAGE "OK"
399 printf " %-70s %-6s\n" $PACKAGE "missing"
407 # configure hugepages
408 function configure_hugepages() {
409 sudo bash -c "echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
410 sudo bash -c "echo 0 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
413 # dump hugepages configuration
414 function hugepages_info() {
415 echo "-------------------------------------------------------------------"
416 head /sys/devices/system/node/node*/hugepages/hugepages*/*
417 echo "-------------------------------------------------------------------"
426 # enter workspace dir
429 # create virtualenv if needed
430 if [ ! -e $VSPERFENV_DIR ] ; then
431 echo "Create VSPERF environment"
432 echo "========================="
433 virtualenv --python=python3 "$VSPERFENV_DIR"
437 # acivate and update virtualenv
438 echo "Update VSPERF environment"
439 echo "========================="
440 source "$VSPERFENV_DIR"/bin/activate
441 pip install -r ./requirements.txt
444 # VERFIY&MERGE job specific - check if required packages are installed
450 # configure hugepages
453 # execute job based on passed parameter
456 echo "================="
457 echo "VSPERF verify job"
458 echo "================="
460 execute_vsperf_pylint_check
462 execute_vsperf_sanity
464 execute_vsperf OVS_with_DPDK_and_vHost_User $1
466 execute_vsperf OVS_vanilla $1
468 execute_vsperf VPP $1
474 echo "================"
475 echo "VSPERF merge job"
476 echo "================"
480 execute_vsperf_sanity
482 execute_vsperf OVS_with_DPDK_and_vHost_User $1
484 execute_vsperf OVS_vanilla $1
486 execute_vsperf VPP $1
492 echo "================"
493 echo "VSPERF daily job"
494 echo "================"
497 execute_vsperf OVS_with_DPDK_and_vHost_User $1
499 execute_vsperf OVS_vanilla $1
501 execute_vsperf VPP $1
503 execute_vsperf SRIOV $1
508 push_results_to_artifactory