tests/ci/yardstick-verify

   1 #!/bin/bash
   2 ##############################################################################
   3 # Copyright (c) 2015 Ericsson AB and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10
  11 #
  12 # Set up the environment and run yardstick test suites.
  13 #
  14 # Example invocation: yardstick-verify -r 10.4.4.4 suite1.yaml suite2.yaml
  15 #
  16 # Openstack credentials must be set and the script must be run from its
  17 # original location in the yardstick repo.
  18 #
  19 # This script is intended to be used by the CI pipeline but it may also
  20 # be invoked manually.
  21 #
  22
  23 SCRIPT=$0
  24 SCRIPT_ARGS=$@
  25
  26 usage()
  27 {
  28     cat << EOF
  29 usage: $0 options [TEST_SUITE ...]
  30
  31 If no test suites are given ping.yaml is run.
  32
  33 OPTIONS:
  34    -h      Show this message
  35    -r      Http target (example: -r 213.77.62.197/results)
  36    -i      Influxdb target (example: -i 127.0.0.1:8086)
  37
  38            Default target is dump to file ($DISPATCHER_FILE_NAME)
  39
  40 EOF
  41 }
  42
  43 DISPATCHER_TYPE=file
  44 DISPATCHER_FILE_NAME="/tmp/yardstick.out"
  45 DISPATCHER_HTTP_TARGET="http://testresults.opnfv.org/test/api/v1/results"
  46 DISPATCHER_INFLUXDB_TARGET=
  47 UCA_HOST="cloud-images.ubuntu.com"
  48 if [ $YARD_IMG_ARCH = "arm64" ]; then
  49     export VIVID_IMG_URL="http://${UCA_HOST}/vivid/current/vivid-server-cloudimg-arm64.tar.gz"
  50     if ! grep -q "Defaults env_keep += \"VIVID_IMG_URL\"" "/etc/sudoers"; then
  51         sudo echo "Defaults env_keep += \"VIVID_IMG_URL\"" >> /etc/sudoers
  52     fi
  53 fi
  54 while getopts "r:i:h" OPTION; do
  55     case $OPTION in
  56         h)
  57             usage
  58             exit 0
  59             ;;
  60         r)
  61             DISPATCHER_TYPE=http
  62             DISPATCHER_HTTP_TARGET=http://${OPTARG}
  63             DISPATCHER_FILE_NAME=
  64             ;;
  65         i)
  66             DISPATCHER_TYPE=influxdb
  67             DISPATCHER_INFLUXDB_TARGET=http://${OPTARG}
  68             DISPATCHER_FILE_NAME=
  69             ;;
  70         *)
  71             echo "${OPTION} is not a valid argument"
  72             exit 1
  73             ;;
  74     esac
  75 done
  76
  77 shift $[OPTIND - 1]
  78 TEST_SUITES=$@
  79
  80 cleanup()
  81 {
  82     echo
  83     echo "========== Cleanup =========="
  84
  85     if ! glance image-list; then
  86         return
  87     fi
  88
  89     for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-trusty-server -e Ubuntu-14.04 \
  90         -e yardstick-vivid-kernel | awk '{print $2}'); do
  91         echo "Deleting image $image..."
  92         glance image-delete $image || true
  93     done
  94
  95     nova flavor-delete yardstick-flavor &> /dev/null || true
  96 }
  97
  98 exitcode=""
  99
 100 error_exit()
 101 {
 102     local rc=$?
 103
 104     if [ -z "$exitcode" ]; then
 105         # In case of recursive traps (!?)
 106         exitcode=$rc
 107     fi
 108
 109     cleanup
 110
 111     echo "Exiting with RC=$exitcode"
 112
 113     exit $exitcode
 114 }
 115
 116 set -o errexit
 117 set -o pipefail
 118
 119 install_yardstick()
 120 {
 121     echo
 122     echo "========== Installing yardstick =========="
 123
 124     # uninstall previous version
 125     pip uninstall -y yardstick || true
 126
 127     # Install yardstick
 128     pip install .
 129 }
 130
 131 install_storperf()
 132 {
 133     # Install Storper on huawei-pod1
 134     if [ "$NODE_NAME" == "huawei-pod1" ]; then
 135         echo
 136         echo "========== Installing storperf =========="
 137
 138         if ! yardstick -d plugin install plugin/CI/storperf.yaml; then
 139             echo "Install storperf plugin FAILED";
 140             exit 1
 141         fi
 142
 143     fi
 144 }
 145
 146 remove_storperf()
 147 {
 148     # remove Storper from huawei-pod1
 149     if [ "$NODE_NAME" == "huawei-pod1" ]; then
 150         echo
 151         echo "========== Removing storperf =========="
 152
 153         if ! yardstick -d plugin remove plugin/CI/storperf.yaml; then
 154             echo "Remove storperf plugin FAILED";
 155             exit 1
 156         fi
 157
 158     fi
 159 }
 160
 161 build_yardstick_image()
 162 {
 163     echo
 164     echo "========== Build yardstick cloud image =========="
 165
 166     local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh"
 167
 168     # Build the image. Retry once if the build fails.
 169     $cmd || $cmd
 170
 171     if [ ! -f $QCOW_IMAGE ]; then
 172         echo "Failed building QCOW image"
 173         exit 1
 174     fi
 175 }
 176
 177 create_nova_flavor()
 178 {
 179     if ! nova flavor-list | grep -q yardstick-flavor; then
 180         echo
 181         echo "========== Create nova flavor =========="
 182         # Create the nova flavor used by some sample test cases
 183         nova flavor-create yardstick-flavor 100 512 3 1
 184         # DPDK-enabled OVS requires guest memory to be backed by large pages
 185         if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then
 186             nova flavor-key yardstick-flavor set hw:mem_page_size=large
 187         fi
 188         # VPP requires guest memory to be backed by large pages
 189         if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
 190             nova flavor-key yardstick-flavor set hw:mem_page_size=large
 191         fi
 192     fi
 193 }
 194
 195 load_cirros_image()
 196 {
 197     echo
 198     echo "========== Loading cirros cloud image =========="
 199
 200     local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img
 201
 202     EXTRA_PARAMS=""
 203     # VPP requires guest memory to be backed by large pages
 204     if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
 205         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
 206     fi
 207
 208     output=$(glance image-create \
 209         --name  cirros-0.3.3 \
 210         --disk-format $DISK_FORMAT \
 211         --container-format bare \
 212         $EXTRA_PARAMS \
 213         --file $image_file)
 214     echo "$output"
 215
 216     CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
 217     if [ -z "$CIRROS_IMAGE_ID" ]; then
 218         echo 'Failed uploading cirros image to cloud'.
 219         exit 1
 220     fi
 221
 222     echo "Cirros image id: $CIRROS_IMAGE_ID"
 223 }
 224
 225 load_ubuntu_image()
 226 {
 227     echo
 228     echo "========== Loading ubuntu cloud image =========="
 229
 230     local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img
 231
 232     EXTRA_PARAMS=""
 233     # VPP requires guest memory to be backed by large pages
 234     if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
 235         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
 236     fi
 237
 238     output=$(glance image-create \
 239         --name Ubuntu-14.04 \
 240         --disk-format qcow2 \
 241         --container-format bare \
 242         $EXTRA_PARAMS \
 243         --file $ubuntu_image_file)
 244     echo "$output"
 245
 246     UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
 247
 248     if [ -z "$UBUNTU_IMAGE_ID" ]; then
 249         echo 'Failed uploading UBUNTU image to cloud'.
 250         exit 1
 251     fi
 252
 253     echo "Ubuntu image id: $UBUNTU_IMAGE_ID"
 254 }
 255
 256 load_yardstick_image()
 257 {
 258     echo
 259     echo "========== Loading yardstick cloud image =========="
 260     EXTRA_PARAMS=""
 261     if [ $YARD_IMG_ARCH = "arm64" ]; then
 262         VIVID_IMAGE="/tmp/vivid-server-cloudimg-arm64.tar.gz"
 263         VIVID_KERNEL="/tmp/vivid-server-cloudimg-arm64-vmlinuz-generic"
 264         cd /tmp
 265         if [ ! -f $VIVID_IMAGE ]; then
 266             wget $VIVID_IMG_URL
 267         fi
 268         if [ ! -f $VIVID_KERNEL ]; then
 269             tar zxf $VIVID_IMAGE $(basename $VIVID_KERNEL)
 270         fi
 271         create_vivid_kernel=$(glance --os-image-api-version 1 image-create \
 272                 --name yardstick-vivid-kernel \
 273                 --is-public true --disk-format qcow2 \
 274                 --container-format bare \
 275                 --file $VIVID_KERNEL)
 276
 277         GLANCE_KERNEL_ID=$(echo "$create_vivid_kernel" | grep " id " | awk '{print $(NF-1)}')
 278         if [ -z "$GLANCE_KERNEL_ID" ]; then
 279             echo 'Failed uploading kernel to cloud'.
 280             exit 1
 281         fi
 282
 283         command_line="root=/dev/vdb1 console=tty0 console=ttyS0 console=ttyAMA0 rw"
 284
 285         EXTRA_PARAMS="--property kernel_id=$GLANCE_KERNEL_ID --property os_command_line=\"$command_line\""
 286
 287         rm -f $VIVID_KERNEL $VIVID_IMAGE
 288         cd $YARDSTICK_REPO_DIR
 289     fi
 290
 291     # VPP requires guest memory to be backed by large pages
 292     if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
 293         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
 294     fi
 295
 296     output=$(eval glance --os-image-api-version 1 image-create \
 297         --name yardstick-trusty-server \
 298         --is-public true --disk-format $DISK_FORMAT \
 299         --container-format bare \
 300         $EXTRA_PARAMS \
 301         --file $QCOW_IMAGE)
 302
 303     echo "$output"
 304
 305     GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
 306
 307     if [ -z "$GLANCE_IMAGE_ID" ]; then
 308         echo 'Failed uploading image to cloud'.
 309         exit 1
 310     fi
 311
 312     sudo rm -f $QCOW_IMAGE
 313
 314     echo "Glance image id: $GLANCE_IMAGE_ID"
 315 }
 316
 317 report(){
 318
 319     echo
 320     echo "========== Reporting Status =========="
 321     curl -i -H 'content-type: application/json' -X POST -d \
 322         "{\"project_name\": \"yardstick\",
 323           \"case_name\": \"scenario_status\",
 324           \"pod_name\":\"${NODE_NAME}\",
 325           \"installer\":\"${INSTALLER_TYPE}\",
 326           \"version\":\"$(basename ${YARDSTICK_BRANCH})\",
 327           \"scenario\":\"${DEPLOY_SCENARIO}\",
 328           \"description\": \"yardstick ci scenario status\",
 329           \"criteria\":\"$1\",
 330           \"start_date\":\"$2\",
 331           \"stop_date\":\"$3\",
 332           \"details\":\"\"}" \
 333           ${DISPATCHER_HTTP_TARGET}
 334 }
 335
 336 run_test()
 337 {
 338     echo
 339     echo "========== Running yardstick test suites =========="
 340
 341     mkdir -p /etc/yardstick
 342
 343     cat << EOF >> /etc/yardstick/yardstick.conf
 344 [DEFAULT]
 345 debug = True
 346 dispatcher = ${DISPATCHER_TYPE}
 347
 348 [dispatcher_file]
 349 file_name = ${DISPATCHER_FILE_NAME}
 350
 351 [dispatcher_http]
 352 timeout = 5
 353 target = ${DISPATCHER_HTTP_TARGET}
 354
 355 [dispatcher_influxdb]
 356 timeout = 5
 357 target = ${DISPATCHER_INFLUXDB_TARGET}
 358 db_name = yardstick
 359 username = opnfv
 360 password = 0pnfv2015
 361 EOF
 362
 363     local failed=0
 364     local start_date
 365     local stop_date
 366
 367     if [ ${#SUITE_FILES[@]} -gt 0 ]; then
 368
 369         start_date=$(date '+%Y-%m-%d %H:%M:%S')
 370         for suite in ${SUITE_FILES[*]}; do
 371
 372             echo "---------------------------"
 373             echo "Running test suite: $suite"
 374             echo "---------------------------"
 375             if ! yardstick task start --suite $suite; then
 376                  echo "test suite $suite FAILED";
 377
 378                 # Mark the test suite failed but continue
 379                 # running the remaining test suites.
 380                 (( ++failed ))
 381             fi
 382             if [ ${DISPATCHER_TYPE} = file ]; then
 383                 echo "---------------------------"
 384                 echo "Dump test suite $suite result"
 385                 echo "---------------------------"
 386                 if [ -f ${DISPATCHER_FILE_NAME} ]; then
 387                     cat ${DISPATCHER_FILE_NAME}
 388                 else
 389                     echo "Test result file ${DISPATCHER_FILE_NAME} is not exist"
 390                 fi
 391             fi
 392         done
 393         stop_date=$(date '+%Y-%m-%d %H:%M:%S')
 394
 395
 396
 397         local scenario_status="SUCCESS"
 398
 399         if [ $failed -gt 0 ]; then
 400             scenario_status="FAILED"
 401         fi
 402
 403         report $scenario_status $start_date $stop_date
 404
 405         if [ $failed -gt 0 ]; then
 406             echo "---------------------------"
 407             echo "$failed out of ${SUITE_FILES[*]} test suites FAILED"
 408             echo "---------------------------"
 409             exit 1
 410         fi
 411
 412     else
 413
 414         echo "---------------------------"
 415         echo "Running samples/ping.yaml  "
 416         echo "---------------------------"
 417
 418         if ! yardstick task start samples/ping.yaml; then
 419             echo "Yardstick test FAILED"
 420             exit 1
 421         fi
 422
 423         if [ ${DISPATCHER_TYPE} = file ]; then
 424             echo "---------------------------"
 425             echo "Dump samples/ping.yaml test result"
 426             echo "---------------------------"
 427             if [ -f ${DISPATCHER_FILE_NAME} ]; then
 428                 cat ${DISPATCHER_FILE_NAME}
 429             else
 430                 echo "Test result file ${DISPATCHER_FILE_NAME} is not exist"
 431             fi
 432         fi
 433
 434     fi
 435
 436 }
 437
 438 main()
 439 {
 440     GITROOT=$(cd $(dirname $0) && git rev-parse --show-toplevel)
 441
 442     cd $GITROOT
 443
 444     export YARDSTICK_VERSION=$(git rev-parse HEAD)
 445
 446     SUITE_FILES=()
 447
 448     # find the test suite files
 449     for suite in $TEST_SUITES; do
 450         if [ -f $suite ]; then
 451             SUITE_FILES+=($suite)
 452         else
 453             tsdir=$GITROOT/tests/opnfv/test_suites
 454             if [ ! -f $tsdir/$suite ]; then
 455                 echo "Test suite \"$suite\" does not exist"
 456                 exit 1
 457             fi
 458             SUITE_FILES+=($tsdir/$suite)
 459         fi
 460     done
 461
 462     echo
 463     echo "========== Running Yardstick CI with following parameters =========="
 464     echo "Script options: ${SCRIPT} $SCRIPT_ARGS"
 465     echo "Dispatcher: ${DISPATCHER_TYPE} ${DISPATCHER_FILE_NAME}"
 466     echo "YARDSTICK_VERSION: ${YARDSTICK_VERSION}"
 467     echo "Number of test suites: ${#SUITE_FILES[@]}"
 468     for suite in ${SUITE_FILES[*]}; do
 469         echo "     $suite"
 470     done
 471     echo
 472
 473     # check if some necessary variables is set
 474     if [ -z "$OS_AUTH_URL" ]; then
 475         echo "OS_AUTH_URL is unset or empty"
 476         exit 1
 477     fi
 478
 479     echo "OS_AUTH_URL is $OS_AUTH_URL"
 480     echo
 481
 482     # check OpenStack services
 483     echo "Checking OpenStack services:"
 484     for cmd in "glance image-list" "nova list" "heat stack-list"; do
 485         echo "  checking ${cmd/%\ */} ..."
 486         if ! $cmd >/dev/null; then
 487             echo "error: command \"$cmd\" failed"
 488             exit 1
 489         fi
 490     done
 491
 492     echo
 493     echo "Checking for External network:"
 494     for net in $(neutron net-list --router:external True -c name -f value); do
 495         echo "  external network: $net"
 496     done
 497
 498     # install yardstick
 499     install_yardstick
 500
 501     cleanup
 502
 503     trap "error_exit" EXIT SIGTERM
 504
 505     QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img"
 506
 507     if [ "$DEPLOY_SCENARIO" == "os-nosdn-lxd-ha" -o "$DEPLOY_SCENARIO" == "os-nosdn-lxd-noha" ]; then
 508         DISK_FORMAT="raw"
 509     else
 510         DISK_FORMAT="qcow2"
 511     fi
 512
 513     build_yardstick_image
 514     load_yardstick_image
 515     if [ $YARD_IMG_ARCH = "arm64" ]; then
 516         sed -i 's/image: cirros-0.3.3/image: TestVM/g' tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml \
 517         samples/ping.yaml
 518         #We have overlapping IP with the real network
 519         for filename in tests/opnfv/test_cases/*; do
 520             sed -i "s/cidr: '10.0.1.0\/24'/cidr: '10.3.1.0\/24'/g" $filename
 521         done
 522     else
 523         load_cirros_image
 524         load_ubuntu_image
 525     fi
 526     create_nova_flavor
 527     install_storperf
 528     run_test
 529     remove_storperf
 530 }
 531
 532 main