Merge "Bugfix: modify comment bugs in TC055 and TC075"
[yardstick.git] / tests / ci / yardstick-verify
1 #!/bin/bash
2 ##############################################################################
3 # Copyright (c) 2015 Ericsson AB and others.
4 #
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ##############################################################################
10
11 #
12 # Set up the environment and run yardstick test suites.
13 #
14 # Example invocation: yardstick-verify -r 10.4.4.4 suite1.yaml suite2.yaml
15 #
16 # Openstack credentials must be set and the script must be run from its
17 # original location in the yardstick repo.
18 #
19 # This script is intended to be used by the CI pipeline but it may also
20 # be invoked manually.
21 #
22
23 SCRIPT=$0
24 SCRIPT_ARGS=$@
25
26 usage()
27 {
28     cat << EOF
29 usage: $0 options [TEST_SUITE ...]
30
31 If no test suites are given ping.yaml is run.
32
33 OPTIONS:
34    -h      Show this message
35    -r      Http target (example: -r 213.77.62.197/results)
36    -i      Influxdb target (example: -i 127.0.0.1:8086)
37
38            Default target is dump to file ($DISPATCHER_FILE_NAME)
39
40 EOF
41 }
42
43 DISPATCHER_TYPE=file
44 DISPATCHER_FILE_NAME="/tmp/yardstick.out"
45 DISPATCHER_HTTP_TARGET="http://testresults.opnfv.org/test/api/v1/results"
46 DISPATCHER_INFLUXDB_TARGET=
47 UCA_HOST="cloud-images.ubuntu.com"
48 if [ $YARD_IMG_ARCH = "arm64" ]; then
49     export VIVID_IMG_URL="http://${UCA_HOST}/vivid/current/vivid-server-cloudimg-arm64.tar.gz"
50     if ! grep -q "Defaults env_keep += \"VIVID_IMG_URL\"" "/etc/sudoers"; then
51         sudo echo "Defaults env_keep += \"VIVID_IMG_URL\"" >> /etc/sudoers
52     fi
53 fi
54 while getopts "r:i:h" OPTION; do
55     case $OPTION in
56         h)
57             usage
58             exit 0
59             ;;
60         r)
61             DISPATCHER_TYPE=http
62             DISPATCHER_HTTP_TARGET=http://${OPTARG}
63             DISPATCHER_FILE_NAME=
64             ;;
65         i)
66             DISPATCHER_TYPE=influxdb
67             DISPATCHER_INFLUXDB_TARGET=http://${OPTARG}
68             DISPATCHER_FILE_NAME=
69             ;;
70         *)
71             echo "${OPTION} is not a valid argument"
72             exit 1
73             ;;
74     esac
75 done
76
77 shift $[OPTIND - 1]
78 TEST_SUITES=$@
79
80 cleanup()
81 {
82     echo
83     echo "========== Cleanup =========="
84
85     if ! glance image-list; then
86         return
87     fi
88
89     for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-trusty-server -e Ubuntu-14.04 \
90         -e yardstick-vivid-kernel | awk '{print $2}'); do
91         echo "Deleting image $image..."
92         glance image-delete $image || true
93     done
94
95     nova flavor-delete yardstick-flavor &> /dev/null || true
96 }
97
98 exitcode=""
99
100 error_exit()
101 {
102     local rc=$?
103
104     if [ -z "$exitcode" ]; then
105         # In case of recursive traps (!?)
106         exitcode=$rc
107     fi
108
109     cleanup
110
111     echo "Exiting with RC=$exitcode"
112
113     exit $exitcode
114 }
115
116 set -o errexit
117 set -o pipefail
118
119 install_yardstick()
120 {
121     echo
122     echo "========== Installing yardstick =========="
123
124     # uninstall previous version
125     pip uninstall -y yardstick || true
126
127     # Install yardstick
128     pip install .
129 }
130
131 install_storperf()
132 {
133     # Install Storper on huawei-pod1
134     if [ "$NODE_NAME" == "huawei-pod1" ]; then
135         echo
136         echo "========== Installing storperf =========="
137
138         if ! yardstick -d plugin install plugin/CI/storperf.yaml; then
139             echo "Install storperf plugin FAILED";
140             exit 1
141         fi
142
143     fi
144 }
145
146 remove_storperf()
147 {
148     # remove Storper from huawei-pod1
149     if [ "$NODE_NAME" == "huawei-pod1" ]; then
150         echo
151         echo "========== Removing storperf =========="
152
153         if ! yardstick -d plugin remove plugin/CI/storperf.yaml; then
154             echo "Remove storperf plugin FAILED";
155             exit 1
156         fi
157
158     fi
159 }
160
161 build_yardstick_image()
162 {
163     echo
164     echo "========== Build yardstick cloud image =========="
165
166     local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh"
167
168     # Build the image. Retry once if the build fails.
169     $cmd || $cmd
170
171     if [ ! -f $QCOW_IMAGE ]; then
172         echo "Failed building QCOW image"
173         exit 1
174     fi
175 }
176
177 create_nova_flavor()
178 {
179     if ! nova flavor-list | grep -q yardstick-flavor; then
180         echo
181         echo "========== Create nova flavor =========="
182         # Create the nova flavor used by some sample test cases
183         nova flavor-create yardstick-flavor 100 512 3 1
184         # DPDK-enabled OVS requires guest memory to be backed by large pages
185         if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then
186             nova flavor-key yardstick-flavor set hw:mem_page_size=large
187         fi
188     fi
189 }
190
191 load_cirros_image()
192 {
193     echo
194     echo "========== Loading cirros cloud image =========="
195
196     local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img
197
198     output=$(glance image-create \
199         --name  cirros-0.3.3 \
200         --disk-format $DISK_FORMAT \
201         --container-format bare \
202         --file $image_file)
203     echo "$output"
204
205     CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
206     if [ -z "$CIRROS_IMAGE_ID" ]; then
207         echo 'Failed uploading cirros image to cloud'.
208         exit 1
209     fi
210
211     echo "Cirros image id: $CIRROS_IMAGE_ID"
212 }
213
214 load_ubuntu_image()
215 {
216     echo
217     echo "========== Loading ubuntu cloud image =========="
218
219     local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img
220
221     output=$(glance image-create \
222         --name Ubuntu-14.04 \
223         --disk-format qcow2 \
224         --container-format bare \
225         --file $ubuntu_image_file)
226     echo "$output"
227
228     UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
229
230     if [ -z "$UBUNTU_IMAGE_ID" ]; then
231         echo 'Failed uploading UBUNTU image to cloud'.
232         exit 1
233     fi
234
235     echo "Ubuntu image id: $UBUNTU_IMAGE_ID"
236 }
237
238 load_yardstick_image()
239 {
240     echo
241     echo "========== Loading yardstick cloud image =========="
242     EXTRA_PARAMS=""
243     if [ $YARD_IMG_ARCH = "arm64" ]; then
244         VIVID_IMAGE="/tmp/vivid-server-cloudimg-arm64.tar.gz"
245         VIVID_KERNEL="/tmp/vivid-server-cloudimg-arm64-vmlinuz-generic"
246         cd /tmp
247         if [ ! -f $VIVID_IMAGE ]; then
248             wget $VIVID_IMG_URL
249         fi
250         if [ ! -f $VIVID_KERNEL ]; then
251             tar zxf $VIVID_IMAGE $(basename $VIVID_KERNEL)
252         fi
253         create_vivid_kernel=$(glance --os-image-api-version 1 image-create \
254                 --name yardstick-vivid-kernel \
255                 --is-public true --disk-format qcow2 \
256                 --container-format bare \
257                 --file $VIVID_KERNEL)
258
259         GLANCE_KERNEL_ID=$(echo "$create_vivid_kernel" | grep " id " | awk '{print $(NF-1)}')
260         if [ -z "$GLANCE_KERNEL_ID" ]; then
261             echo 'Failed uploading kernel to cloud'.
262             exit 1
263         fi
264
265         command_line="root=/dev/vdb1 console=tty0 console=ttyS0 console=ttyAMA0 rw"
266
267         EXTRA_PARAMS="--property kernel_id=$GLANCE_KERNEL_ID --property os_command_line=\"$command_line\""
268
269         rm -f $VIVID_KERNEL $VIVID_IMAGE
270         cd $YARDSTICK_REPO_DIR
271     fi
272
273     output=$(eval glance --os-image-api-version 1 image-create \
274         --name yardstick-trusty-server \
275         --is-public true --disk-format $DISK_FORMAT \
276         --container-format bare \
277         $EXTRA_PARAMS \
278         --file $QCOW_IMAGE)
279
280     echo "$output"
281
282     GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
283
284     if [ -z "$GLANCE_IMAGE_ID" ]; then
285         echo 'Failed uploading image to cloud'.
286         exit 1
287     fi
288
289     sudo rm -f $QCOW_IMAGE
290
291     echo "Glance image id: $GLANCE_IMAGE_ID"
292 }
293
294 report(){
295
296     echo
297     echo "========== Reporting Status =========="
298
299     curl -i -H 'content-type: application/json' -X POST -d \
300         "{\"project_name\": \"yardstick\",
301           \"case_name\": \"scenario_status\",
302           \"pod_name\":\"${NODE_NAME}\",
303           \"installer\":\"${INSTALLER_TYPE}\",
304           \"version\":\"${YARDSTICK_BRANCH}\",
305           \"scenario\":\"${DEPLOY_SCENARIO}\",
306           \"description\": \"yardstick ci scenario status\",
307           \"start_date\":\"$2\",
308           \"criteria\":\"$1\",
309           \"stop_date\":\"$3\",
310           \"details\":\"\"}" \
311           ${DISPATCHER_HTTP_TARGET}
312 }
313
314 run_test()
315 {
316     echo
317     echo "========== Running yardstick test suites =========="
318
319     mkdir -p /etc/yardstick
320
321     cat << EOF >> /etc/yardstick/yardstick.conf
322 [DEFAULT]
323 debug = True
324 dispatcher = ${DISPATCHER_TYPE}
325
326 [dispatcher_file]
327 file_name = ${DISPATCHER_FILE_NAME}
328
329 [dispatcher_http]
330 timeout = 5
331 target = ${DISPATCHER_HTTP_TARGET}
332
333 [dispatcher_influxdb]
334 timeout = 5
335 target = ${DISPATCHER_INFLUXDB_TARGET}
336 db_name = yardstick
337 username = opnfv
338 password = 0pnfv2015
339 EOF
340
341     local failed=0
342     local start_date
343     local stop_date
344
345     if [ ${#SUITE_FILES[@]} -gt 0 ]; then
346
347         start_date=$(date '+%Y-%m-%d %H:%M:%S')
348         for suite in ${SUITE_FILES[*]}; do
349
350             echo "---------------------------"
351             echo "Running test suite: $suite"
352             echo "---------------------------"
353             if ! yardstick task start --suite $suite; then
354                  echo "test suite $suite FAILED";
355
356                 # Mark the test suite failed but continue
357                 # running the remaining test suites.
358                 (( ++failed ))
359             fi
360             if [ ${DISPATCHER_TYPE} = file ]; then
361                 echo "---------------------------"
362                 echo "Dump test suite $suite result"
363                 echo "---------------------------"
364                 if [ -f ${DISPATCHER_FILE_NAME} ]; then
365                     cat ${DISPATCHER_FILE_NAME}
366                 else
367                     echo "Test result file ${DISPATCHER_FILE_NAME} is not exist"
368                 fi
369             fi
370         done
371         stop_date=$(date '+%Y-%m-%d %H:%M:%S')
372
373
374
375         local scenario_status="SUCCESS"
376
377         if [ $failed -gt 0 ]; then
378             scenario_status="FAILED"
379         fi
380
381         report $scenario_status $start_date $stop_date
382
383         if [ $failed -gt 0 ]; then
384             echo "---------------------------"
385             echo "$failed out of ${SUITE_FILES[*]} test suites FAILED"
386             echo "---------------------------"
387             exit 1
388         fi
389
390     else
391
392         echo "---------------------------"
393         echo "Running samples/ping.yaml  "
394         echo "---------------------------"
395
396         if ! yardstick task start samples/ping.yaml; then
397             echo "Yardstick test FAILED"
398             exit 1
399         fi
400
401         if [ ${DISPATCHER_TYPE} = file ]; then
402             echo "---------------------------"
403             echo "Dump samples/ping.yaml test result"
404             echo "---------------------------"
405             if [ -f ${DISPATCHER_FILE_NAME} ]; then
406                 cat ${DISPATCHER_FILE_NAME}
407             else
408                 echo "Test result file ${DISPATCHER_FILE_NAME} is not exist"
409             fi
410         fi
411
412     fi
413
414 }
415
416 main()
417 {
418     GITROOT=$(cd $(dirname $0) && git rev-parse --show-toplevel)
419
420     cd $GITROOT
421
422     export YARDSTICK_VERSION=$(git rev-parse HEAD)
423
424     SUITE_FILES=()
425
426     # find the test suite files
427     for suite in $TEST_SUITES; do
428         if [ -f $suite ]; then
429             SUITE_FILES+=($suite)
430         else
431             tsdir=$GITROOT/tests/opnfv/test_suites
432             if [ ! -f $tsdir/$suite ]; then
433                 echo "Test suite \"$suite\" does not exist"
434                 exit 1
435             fi
436             SUITE_FILES+=($tsdir/$suite)
437         fi
438     done
439
440     echo
441     echo "========== Running Yardstick CI with following parameters =========="
442     echo "Script options: ${SCRIPT} $SCRIPT_ARGS"
443     echo "Dispatcher: ${DISPATCHER_TYPE} ${DISPATCHER_FILE_NAME}"
444     echo "YARDSTICK_VERSION: ${YARDSTICK_VERSION}"
445     echo "Number of test suites: ${#SUITE_FILES[@]}"
446     for suite in ${SUITE_FILES[*]}; do
447         echo "     $suite"
448     done
449     echo
450
451     # check if some necessary variables is set
452     if [ -z "$OS_AUTH_URL" ]; then
453         echo "OS_AUTH_URL is unset or empty"
454         exit 1
455     fi
456
457     echo "OS_AUTH_URL is $OS_AUTH_URL"
458     echo
459
460     # check OpenStack services
461     echo "Checking OpenStack services:"
462     for cmd in "glance image-list" "nova list" "heat stack-list"; do
463         echo "  checking ${cmd/%\ */} ..."
464         if ! $cmd >/dev/null; then
465             echo "error: command \"$cmd\" failed"
466             exit 1
467         fi
468     done
469
470     echo
471     echo "Checking for External network:"
472     for net in $(neutron net-list --router:external True -c name -f value); do
473         echo "  external network: $net"
474     done
475
476     # install yardstick
477     install_yardstick
478
479     cleanup
480
481     trap "error_exit" EXIT SIGTERM
482
483     QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img"
484
485     if [ "$DEPLOY_SCENARIO" == "os-nosdn-lxd-ha" -o "$DEPLOY_SCENARIO" == "os-nosdn-lxd-noha" ]; then
486         DISK_FORMAT="raw"
487     else
488         DISK_FORMAT="qcow2"
489     fi
490
491     build_yardstick_image
492     load_yardstick_image
493     if [ $YARD_IMG_ARCH = "arm64" ]; then
494         sed -i 's/image: cirros-0.3.3/image: TestVM/g' tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml \
495         samples/ping.yaml
496         #We have overlapping IP with the real network
497         for filename in tests/opnfv/test_cases/*; do
498             sed -i "s/cidr: '10.0.1.0\/24'/cidr: '10.3.1.0\/24'/g" $filename
499         done
500     else
501         load_cirros_image
502         load_ubuntu_image
503     fi
504     create_nova_flavor
505     install_storperf
506     run_test
507     remove_storperf
508 }
509
510 main