3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # VSPERF nightly build execution script
20 # build-vsperf.sh job_type
21 # where job_type is one of "verify", "merge", "daily"
29 EXIT_SANITY_TC_FAILED=2
31 EXIT_NO_TEST_REPORT_LOG_DIR=11
38 LOG_FILE_PREFIX="/tmp/vsperf_build"
39 DATE=$(date -u +"%Y-%m-%d_%H-%M-%S")
40 BRANCH=${GIT_BRANCH##*/}
41 VSPERFENV_DIR="$HOME/vsperfenv"
43 # CI job specific configuration
44 # VERIFY - run basic set of TCs with default settings
45 TESTCASES_VERIFY="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
46 TESTPARAM_VERIFY="--integration"
47 # MERGE - run selected TCs with default settings
48 TESTCASES_MERGE="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
49 TESTPARAM_MERGE="--integration"
50 # DAILY - run selected TCs for defined packet sizes
51 TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back'
52 TESTPARAM_DAILY='--test-params pkt_sizes=64,128,512,1024,1518'
53 # check if user config file exists if not then we will use default settings
54 if [ -f $HOME/vsperf-${BRANCH}.conf ] ; then
55 # branch specific config was found
56 CONF_FILE="--conf-file ${HOME}/vsperf-${BRANCH}.conf"
58 if [ -f $HOME/vsperf.conf ] ; then
59 CONF_FILE="--conf-file ${HOME}/vsperf.conf"
65 # Test report related configuration
66 TEST_REPORT_PARTIAL="*_test_report.rst"
67 TEST_REPORT_DIR="${WORKSPACE}/docs/results"
68 TEST_REPORT_INDEX="${TEST_REPORT_DIR}/index.rst"
69 TEST_REPORT_LINK_OLD="https://wiki.opnfv.org/wiki/vsperf_results"
70 TEST_REPORT_FILE="${WORKSPACE}/docs_output/results/results.pdf"
71 TEST_REPORT_TARBALL="vswitchperf_logs_${DATE}.tar.gz"
73 if [[ "x${BRANCH}" == "xmaster" ]]; then
74 TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$DATE/${TEST_REPORT_TARBALL}"
76 TEST_REPORT_LINK_NEW="https://artifacts.opnfv.org/logs/$PROJECT/$NODE_NAME/$BRANCH/$DATE/${TEST_REPORT_TARBALL}"
79 TEST_REPORT_LOG_DIR="${HOME}/opnfv/$PROJECT/results/$BRANCH"
85 # terminate vsperf and all its utilities
86 # it is expected that vsperf is the only python3 app
87 # and no other ovs or qemu instances are running
91 function terminate_vsperf() {
92 sudo pkill stress &> /dev/null
93 sudo pkill python3 &> /dev/null
94 sudo killall -9 qemu-system-x86_64 &> /dev/null
96 # sometimes qemu resists to terminate, so wait a bit and kill it again
97 if pgrep qemu-system-x86_64 &> /dev/null ; then
99 sudo killall -9 qemu-system-x86_64 &> /dev/null
103 sudo pkill ovs-vswitchd &> /dev/null
105 sudo pkill ovsdb-server &> /dev/null
109 # check and print testcase execution status
111 # $1 - directory with results
112 function print_results() {
113 for i in $TESTCASES ; do
114 RES_FILE=`ls -1 $1 | egrep "result_${i}_[0-9a-zA-Z\-]+.csv"`
116 if [ "x$RES_FILE" != "x" -a -e "${1}/${RES_FILE}" ]; then
117 if grep ^FAILED "${1}/${RES_FILE}" &> /dev/null ; then
118 printf " %-70s %-6s\n" "result_${i}" "FAILED"
121 printf " %-70s %-6s\n" "result_${i}" "OK"
124 printf " %-70s %-6s\n" "result_${i}" "FAILED"
130 # execute tests and display results
132 # $1 - vswitch and vnf combination, one of OVS_vanilla, OVS_with_DPDK_and_vHost_User
133 # $2 - CI job type, one of verify, merge, daily
134 function execute_vsperf() {
136 # figure out list of TCs and execution parameters
139 TESTPARAM=$TESTPARAM_VERIFY
140 TESTCASES=$TESTCASES_VERIFY
143 TESTPARAM=$TESTPARAM_MERGE
144 TESTCASES=$TESTCASES_MERGE
147 # by default use daily build and upload results to the OPNFV databse
148 TESTPARAM=$TESTPARAM_DAILY
149 TESTCASES=$TESTCASES_DAILY
150 OPNFVPOD="--opnfvpod=$NODE_NAME"
155 echo -e "\nExecution of VSPERF for $1"
157 DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
161 # figure out log file name
162 LOG_SUBDIR="OvsVanilla"
163 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
165 echo " $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE"
166 $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
169 # figure out log file name
170 LOG_SUBDIR="OvsDpdkVhost"
171 LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
173 echo " $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
174 $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
178 # evaluation of results
179 echo -e "\nResults for $1"
180 RES_DIR="/$(grep "Creating result directory" $LOG_FILE | cut -d'/' -f2-)"
181 if [[ "/" == "${RES_DIR}" ]] ; then
182 echo "FAILURE: Results are not available."
183 echo "-------------------------------------------------------------------"
185 echo "-------------------------------------------------------------------"
186 exit $EXIT_NO_RESULTS
188 print_results "${RES_DIR}"
189 if [ "$EXIT" -eq "$EXIT_TC_FAILED" ] ; then
190 echo "-------------------------------------------------------------------"
192 echo "-------------------------------------------------------------------"
196 # show detailed result figures
197 for md_file in $(grep '\.md"$' $LOG_FILE | cut -d'"' -f2); do
198 # TC resut file header
199 echo -e "\n-------------------------------------------------------------------"
201 echo -e "-------------------------------------------------------------------\n"
203 sed -n '/^- Test ID/,/Bidirectional/{/Packet size/b;p;/Bidirectional/q};/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file
205 sed -n '/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file | grep -v "Unknown" | cat -s
208 # add test results into the final doc template
209 for report in ${RES_DIR}/${TEST_REPORT_PARTIAL} ; do
210 # modify link to the artifactory with test report and logs
211 if [ -f $report ] ; then
212 sed -i -e "s,$TEST_REPORT_LINK_OLD,$TEST_REPORT_LINK_NEW," "$report"
213 cp $report $TEST_REPORT_DIR
214 echo " $(basename $report)" >> $TEST_REPORT_INDEX
218 # copy logs into dedicated directory
219 mkdir ${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}
220 [ -f "$LOG_FILE" ] && cp -a "${LOG_FILE}" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
221 [ -d "$RES_DIR" ] && cp -ar "$RES_DIR" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
224 # generates final test_report in PDF and HTML formats
225 function generate_report() {
227 # prepare final tarball with all logs...
228 tar --exclude "${TEST_REPORT_TARBALL}" -czf "${TEST_REPORT_LOG_DIR}/${TEST_REPORT_TARBALL}" $(find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d)
229 # ...and remove original log files
230 find "${TEST_REPORT_LOG_DIR}" -mindepth 1 -maxdepth 1 -type d -exec rm -rf \{\} \;
232 # clone opnfvdocs repository
233 echo "Cloning opnfvdocs repository..."
234 [ -d opnfvdocs ] && rm -rf opnfvdocs
235 git clone https://gerrit.opnfv.org/gerrit/opnfvdocs &> /dev/null
237 # generate final docs with test results
238 echo "Generating test report..."
239 sed -ie 's,python ,python2 ,g' ./opnfvdocs/scripts/docs-build.sh
240 OPNFVDOCS_DIR='./opnfvdocs' ./opnfvdocs/scripts/docs-build.sh &> /dev/null
242 # store PDF with test results into dedicated directory
243 if [ -f $TEST_REPORT_FILE ] ; then
244 cp -a $TEST_REPORT_FILE $TEST_REPORT_LOG_DIR
245 echo "Final test report has been created."
247 echo "FAILURE: Generation of final test report has failed."
251 # pushes test report and logs collected during test execution into artifactory
252 function push_results_to_artifactory() {
253 # clone releng repository
254 echo "Cloning releng repository..."
255 [ -d releng ] && rm -rf releng
256 git clone https://gerrit.opnfv.org/gerrit/releng &> /dev/null
258 echo "Pushing results and logs into artifactory..."
259 . ./releng/utils/push-test-logs.sh "$DATE"
261 # enter workspace as it could be modified by 3rd party script
265 # removes any local changes of repository
267 echo "Cleaning up..."
271 # prepares directory for logs collection and removes old logs
272 function initialize_logdir() {
273 if [[ "x$TEST_REPORT_LOG_DIR" == "x" ]] ; then
274 echo "FAILURE: Logging directory is not defined. Logs and report cannot be published!"
275 exit $EXIT_NO_TEST_REPORT_LOG_DIR
277 # remove TEST_REPORT_LOG_DIR if it exists
278 if [ -e $TEST_REPORT_LOG_DIR ] ; then
279 if [ -f $TEST_REPORT_LOG_DIR ] ; then
280 rm $TEST_REPORT_LOG_DIR
282 rm -rf ${TEST_REPORT_LOG_DIR}
285 # create TEST_REPORT_LOG_DIR
286 mkdir -p $TEST_REPORT_LOG_DIR
290 # verify basic vsperf functionality
291 function execute_vsperf_sanity() {
292 DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
293 LOG_FILE="${LOG_FILE_PREFIX}_sanity_${DATE_SUFFIX}.log"
294 echo "Execution of VSPERF sanity checks:"
295 for PARAM in '--version' '--help' '--list-trafficgens' '--list-collectors' '--list-vswitches' '--list-fwdapps' '--list-vnfs' '--list-settings' '--list' '--integration --list'; do
296 echo -e "-------------------------------------------------------------------" >> $LOG_FILE
297 echo "$VSPERF_BIN $PARAM $CONF_FILE" >> $LOG_FILE
298 echo -e "-------------------------------------------------------------------" >> $LOG_FILE
299 $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE
300 if $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE ; then
301 printf " %-70s %-6s\n" "$VSPERF_BIN $PARAM" "OK"
303 printf " %-70s %-6s\n" "$VSPERF_BIN $PARAM" "FAILED"
304 EXIT=$EXIT_SANITY_TC_FAILED
308 echo "Sanity log file $LOG_FILE"
309 if [ "$EXIT" -ne "0" ] ; then
310 echo "-------------------------------------------------------------------"
312 echo "-------------------------------------------------------------------"
316 # check and install required packages at nodes running VERIFY and MERGE jobs
317 function dependencies_check() {
319 if [ $ID == "ubuntu" ] ; then
320 echo "Dependencies check"
321 echo "=================="
322 for PACKAGE in "python3-tk" ; do
323 if dpkg -s $PACKAGE &> /dev/null ; then
324 printf " %-70s %-6s\n" $PACKAGE "OK"
326 printf " %-70s %-6s\n" $PACKAGE "missing"
327 sudo apt-get install -y $PACKAGE
340 # enter workspace dir
343 # create virtualenv if needed
344 if [ ! -e $VSPERFENV_DIR ] ; then
345 echo "Create VSPERF environment"
346 echo "========================="
347 virtualenv --python=python3 "$VSPERFENV_DIR"
351 # acivate and update virtualenv
352 echo "Update VSPERF environment"
353 echo "========================="
354 source "$VSPERFENV_DIR"/bin/activate
355 pip install -r ./requirements.txt
358 # VERFIY&MERGE job specific - check if required packages are installed
364 # execute job based on passed parameter
367 echo "================="
368 echo "VSPERF verify job"
369 echo "================="
372 execute_vsperf_sanity
374 execute_vsperf OVS_with_DPDK_and_vHost_User $1
376 execute_vsperf OVS_vanilla $1
381 echo "================"
382 echo "VSPERF merge job"
383 echo "================"
386 execute_vsperf_sanity
388 execute_vsperf OVS_with_DPDK_and_vHost_User $1
390 execute_vsperf OVS_vanilla $1
395 echo "================"
396 echo "VSPERF daily job"
397 echo "================"
400 execute_vsperf OVS_with_DPDK_and_vHost_User $1
402 execute_vsperf OVS_vanilla $1
407 push_results_to_artifactory