Merge "Provide libvirt credentials"
[releng.git] / jjb / dovetail / dovetail-run.sh
1 #!/bin/bash
2 ##############################################################################
3 # Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 #the noun INSTALLER is used in community, here is just the example to run.
11 #multi-platforms are supported.
12
13 set -e
14 [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null"
15
16 DEPLOY_TYPE=baremetal
17 [[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt
18
19 DOVETAIL_HOME=${WORKSPACE}/ovp
20 [ -d ${DOVETAIL_HOME} ] && sudo rm -rf ${DOVETAIL_HOME}
21
22 mkdir -p ${DOVETAIL_HOME}
23
24 DOVETAIL_CONFIG=${DOVETAIL_HOME}/pre_config
25 mkdir -p ${DOVETAIL_CONFIG}
26
27 DOVETAIL_IMAGES=${DOVETAIL_HOME}/images
28 mkdir -p ${DOVETAIL_IMAGES}
29
30 OPENRC=${DOVETAIL_CONFIG}/env_config.sh
31 CACERT=${DOVETAIL_CONFIG}/os_cacert
32
33 ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
34
35 sshkey=""
36
37 check_file_exists() {
38     if [[ -f $1 ]]; then
39         return 0
40     else
41         return 1
42     fi
43 }
44
45 get_cred_file_with_scripts() {
46     echo "INFO: clone releng repo..."
47     releng_repo=${WORKSPACE}/releng
48     [ -d ${releng_repo} ] && sudo rm -rf ${releng_repo}
49     git clone https://gerrit.opnfv.org/gerrit/releng ${releng_repo} >/dev/null
50
51     echo "INFO: clone pharos repo..."
52     pharos_repo=${WORKSPACE}/pharos
53     [ -d ${pharos_repo} ] && sudo rm -rf ${pharos_repo}
54     git clone https://git.opnfv.org/pharos ${pharos_repo} >/dev/null
55
56     echo "INFO: SUT branch is $SUT_BRANCH"
57     echo "INFO: dovetail branch is $BRANCH"
58     BRANCH_BACKUP=$BRANCH
59     export BRANCH=$SUT_BRANCH
60     cmd="${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${CACERT} >${redirect}"
61     echo "INFO: cmd is ${cmd}"
62     ${cmd}
63     export BRANCH=$BRANCH_BACKUP
64 }
65
66 get_apex_cred_file() {
67     instack_mac=$(sudo virsh domiflist undercloud | grep default | \
68                   grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+")
69     INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'})
70     sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa"
71     if [[ -n $(sudo iptables -L FORWARD |grep "REJECT"|grep "reject-with icmp-port-unreachable") ]]; then
72         #note: this happens only in opnfv-lf-pod1
73         sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
74         sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
75     fi
76     get_cred_file_with_scripts
77 }
78
79 get_compass_cred_file() {
80     get_cred_file_with_scripts
81 }
82
83 get_fuel_cred_file() {
84     get_cred_file_with_scripts
85 }
86
87 get_joid_cred_file() {
88     # If production lab then creds may be retrieved dynamically
89     # creds are on the jumphost, always in the same folder
90     sudo cp $LAB_CONFIG/admin-openrc $OPENRC
91 }
92
93 change_cred_file_cacert_path() {
94     check_file_exists ${CACERT}
95     if [[ $? == 0 ]]; then
96         echo "INFO: set ${INSTALLER_TYPE} openstack cacert file to be ${CACERT}"
97         if [[ ${INSTALLER_TYPE} == "compass" ]]; then
98             echo "export OS_CACERT=${CACERT}" >> ${OPENRC}
99         elif [[ ${INSTALLER_TYPE} == "fuel" ]]; then
100             sed -i "s#/etc/ssl/certs/mcp_os_cacert#${CACERT}#g" ${OPENRC}
101         fi
102     else
103         echo "ERROR: cannot find file $OPENRC. Please check if it is existing."
104         sudo ls -al ${DOVETAIL_CONFIG}
105         exit 1
106     fi
107 }
108
109 change_cred_file_ext_net() {
110     check_file_exists ${OPENRC}
111     if [[ $? == 0 ]]; then
112         echo "export EXTERNAL_NETWORK=${EXTERNAL_NETWORK}" >> ${OPENRC}
113     else
114         echo "ERROR: cannot find file $OPENRC. Please check if it is existing."
115         sudo ls -al ${DOVETAIL_CONFIG}
116         exit 1
117     fi
118 }
119
120 get_cred_file() {
121     if [[ ${INSTALLER_TYPE} == 'apex' ]]; then
122         get_apex_cred_file
123     elif [[ ${INSTALLER_TYPE} == 'compass' ]]; then
124         get_compass_cred_file
125     elif [[ ${INSTALLER_TYPE} == 'fuel' ]]; then
126         get_fuel_cred_file
127     elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then
128         get_joid_cred_file
129     fi
130
131     check_file_exists ${OPENRC}
132     if [[ $? == 0 ]]; then
133         echo "INFO: original openstack credentials file is"
134         cat $OPENRC
135         echo "INFO: change cacert file path in credentials file"
136         change_cred_file_cacert_path
137         echo "INFO: set external network in credentials file"
138         change_cred_file_ext_net
139         echo "INFO: final openstack credentials file is"
140         cat $OPENRC
141     else
142         echo "ERROR: cannot find file $OPENRC. Please check if it is existing."
143         sudo ls -al ${DOVETAIL_CONFIG}
144         exit 1
145     fi
146 }
147
148 get_cred_file
149
150 # These packages are used for parsing yaml files and decrypting ipmi user and password.
151 sudo pip install shyaml
152 sudo yum install -y rubygems || sudo apt-get install -y ruby
153 sudo gem install hiera-eyaml
154
155 if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then
156     compass_repo=${WORKSPACE}/compass4nfv/
157     git clone https://github.com/opnfv/compass4nfv.git ${compass_repo} >/dev/null
158     scenario_file=${compass_repo}/deploy/conf/hardware_environment/$NODE_NAME/os-nosdn-nofeature-ha.yml
159     ipmiIp=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiIp)
160     ipmiPass=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiPass)
161     ipmiUser=root
162     jumpserver_ip=$(ifconfig | grep -A 5 docker0 | grep "inet addr" | cut -d ':' -f 2 | cut -d ' ' -f 1)
163
164     cat << EOF >${DOVETAIL_CONFIG}/pod.yaml
165 nodes:
166 - {ip: ${jumpserver_ip}, name: node0, password: root, role: Jumpserver, user: root}
167 - {ip: 10.1.0.50, name: node1, password: root, role: controller, user: root,
168    ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}}
169 - {ip: 10.1.0.51, name: node2, password: root, role: controller, user: root}
170 - {ip: 10.1.0.52, name: node3, password: root, role: controller, user: root}
171 - {ip: 10.1.0.53, name: node4, password: root, role: compute, user: root}
172 - {ip: 10.1.0.54, name: node5, password: root, role: compute, user: root}
173
174 EOF
175 fi
176
177 if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then
178     fuel_ctl_ssh_options="${ssh_options} -i ${SSH_KEY}"
179     ssh_user="ubuntu"
180     fuel_ctl_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
181             "sudo salt 'cfg*' pillar.get _param:openstack_control_address --out text| \
182                 cut -f2 -d' '")
183     fuel_cmp_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
184             "sudo salt 'cmp001*' pillar.get _param:openstack_control_address --out text| \
185                 cut -f2 -d' '")
186     fuel_dbs_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
187             "sudo salt 'dbs01*' pillar.get _param:openstack_database_node01_address --out text| \
188                 cut -f2 -d' '")
189     fuel_msg_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
190             "sudo salt 'msg01*' pillar.get _param:openstack_message_queue_node01_address --out text| \
191                 cut -f2 -d' '")
192     ipmi_index=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
193             "sudo salt 'ctl*' network.ip_addrs cidr=${fuel_ctl_ip} --out text | grep ${fuel_ctl_ip} | cut -c 5")
194
195     organization="$(cut -d'-' -f1 <<< "${NODE_NAME}")"
196     pod_name="$(cut -d'-' -f2 <<< "${NODE_NAME}")"
197     pdf_file=${pharos_repo}/labs/${organization}/${pod_name}.yaml
198     ipmiIp=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.address)
199     ipmiIp="$(cut -d'/' -f1 <<< "${ipmiIp}")"
200     ipmiPass=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.pass)
201     ipmiUser=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.user)
202     [[ $ipmiUser == ENC* ]] && ipmiUser=$(eyaml decrypt -s ${ipmiUser//[[:blank:]]/})
203     [[ $ipmiPass == ENC* ]] && ipmiPass=$(eyaml decrypt -s ${ipmiPass//[[:blank:]]/})
204
205     cat << EOF >${DOVETAIL_CONFIG}/pod.yaml
206 nodes:
207 - {ip: ${INSTALLER_IP}, name: node0, key_filename: /home/opnfv/userconfig/pre_config/id_rsa,
208    role: Jumpserver, user: ${ssh_user}}
209 - {ip: ${fuel_ctl_ip}, name: node1, key_filename: /home/opnfv/userconfig/pre_config/id_rsa,
210    role: controller, user: ${ssh_user}, ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}}
211 - {ip: ${fuel_msg_ip}, name: msg01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}}
212 - {ip: ${fuel_cmp_ip}, name: cmp01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}}
213 - {ip: ${fuel_dbs_ip}, name: dbs01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}}
214 EOF
215 fi
216
217 if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then
218     set +e
219
220     sudo pip install virtualenv
221
222     cd ${releng_repo}/modules
223     sudo virtualenv venv
224     source venv/bin/activate
225     sudo pip install -e ./ >/dev/null
226     sudo pip install netaddr
227
228     if [[ ${INSTALLER_TYPE} == compass ]]; then
229         options="-u root -p root"
230     elif [[ ${INSTALLER_TYPE} == fuel ]]; then
231         options="-u root -p r00tme"
232     elif [[ ${INSTALLER_TYPE} == apex ]]; then
233         options="-u stack -k /root/.ssh/id_rsa"
234     elif [[ ${INSTALLER_TYPE} == daisy ]]; then
235         options="-u root -p r00tme"
236     else
237         echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently."
238         echo "HA test cases may not run properly."
239     fi
240
241     cmd="sudo python ${releng_repo}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \
242          -i ${INSTALLER_IP} ${options} -f ${DOVETAIL_CONFIG}/pod.yaml \
243          -s /home/opnfv/userconfig/pre_config/id_rsa"
244     echo ${cmd}
245     ${cmd}
246
247     deactivate
248
249     set -e
250
251     cd ${WORKSPACE}
252 fi
253
254 if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then
255     sudo chmod 666 ${DOVETAIL_CONFIG}/pod.yaml
256     echo "Adapt process info for $INSTALLER_TYPE ..."
257     if [ "$INSTALLER_TYPE" == "apex" ]; then
258         cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml
259 process_info:
260 - {testcase_name: yardstick.ha.rabbitmq, attack_process: rabbitmq_server}
261 - {testcase_name: yardstick.ha.cinder_api, attack_process: cinder_wsgi}
262 EOF
263     elif [ "$INSTALLER_TYPE" == "fuel" ]; then
264         cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml
265 process_info:
266 - {testcase_name: yardstick.ha.cinder_api, attack_process: cinder-wsgi}
267 - {testcase_name: yardstick.ha.rabbitmq, attack_process: rabbitmq_server, attack_host: msg01}
268 - {testcase_name: yardstick.ha.neutron_l3_agent, attack_process: neutron-l3-agent, attack_host: cmp01}
269 - {testcase_name: yardstick.ha.database, attack_process: mysqld, attack_host: dbs01}
270 EOF
271     elif [ "$INSTALLER_TYPE" == "compass" ]; then
272         cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml
273 process_info:
274 - {testcase_name: yardstick.ha.rabbitmq, attack_process: rabbitmq}
275 EOF
276     fi
277
278     echo "file ${DOVETAIL_CONFIG}/pod.yaml:"
279     cat ${DOVETAIL_CONFIG}/pod.yaml
280 else
281     echo "Error: cannot find file ${DOVETAIL_CONFIG}/pod.yaml. Please check if it is existing."
282     sudo ls -al ${DOVETAIL_CONFIG}
283     echo "HA test cases may not run properly."
284 fi
285
286 if [ "$INSTALLER_TYPE" == "fuel" ]; then
287     if [[ "${SUT_BRANCH}" =~ "danube" ]]; then
288         echo "Fetching id_rsa file from jump_server $INSTALLER_IP..."
289         sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa
290     else
291         cp ${SSH_KEY} ${DOVETAIL_CONFIG}/id_rsa
292     fi
293 fi
294
295 if [ "$INSTALLER_TYPE" == "apex" ]; then
296     echo "Fetching id_rsa file from jump_server $INSTALLER_IP..."
297     sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa
298 fi
299
300 if [ "$INSTALLER_TYPE" == "daisy" ]; then
301     echo "Fetching id_dsa file from jump_server $INSTALLER_IP..."
302     sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_dsa ${DOVETAIL_CONFIG}/id_rsa
303 fi
304
305
306 image_path=${HOME}/opnfv/dovetail/images
307 if [[ ! -d ${image_path} ]]; then
308     mkdir -p ${image_path}
309 fi
310 # sdnvpn test case needs to download this image first before running
311 ubuntu_image=${image_path}/ubuntu-16.04-server-cloudimg-amd64-disk1.img
312 if [[ ! -f ${ubuntu_image} ]]; then
313     echo "Download image ubuntu-16.04-server-cloudimg-amd64-disk1.img ..."
314     wget -q -nc https://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${image_path}
315 fi
316 sudo cp ${ubuntu_image} ${DOVETAIL_IMAGES}
317
318 # yardstick and bottlenecks need to download this image first before running
319 cirros_image=${image_path}/cirros-0.3.5-x86_64-disk.img
320 if [[ ! -f ${cirros_image} ]]; then
321     echo "Download image cirros-0.3.5-x86_64-disk.img ..."
322     wget -q -nc http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img -P ${image_path}
323 fi
324 sudo cp ${cirros_image} ${DOVETAIL_IMAGES}
325
326 # functest needs to download this image first before running
327 cirros_image=${image_path}/cirros-0.4.0-x86_64-disk.img
328 if [[ ! -f ${cirros_image} ]]; then
329     echo "Download image cirros-0.4.0-x86_64-disk.img ..."
330     wget -q -nc http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img -P ${image_path}
331 fi
332 sudo cp ${cirros_image} ${DOVETAIL_IMAGES}
333
334 # snaps_smoke test case needs to download this image first before running
335 ubuntu14_image=${image_path}/ubuntu-14.04-server-cloudimg-amd64-disk1.img
336 if [[ ! -f ${ubuntu14_image} ]]; then
337     echo "Download image ubuntu-14.04-server-cloudimg-amd64-disk1.img ..."
338     wget -q -nc https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img -P ${image_path}
339 fi
340 sudo cp ${ubuntu14_image} ${DOVETAIL_IMAGES}
341
342 # cloudify_ims test case needs to download these 2 images first before running
343 cloudify_image=${image_path}/cloudify-manager-premium-4.0.1.qcow2
344 if [[ ! -f ${cloudify_image} ]]; then
345     echo "Download image cloudify-manager-premium-4.0.1.qcow2 ..."
346     wget -q -nc http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 -P ${image_path}
347 fi
348 sudo cp ${cloudify_image} ${DOVETAIL_IMAGES}
349 trusty_image=${image_path}/trusty-server-cloudimg-amd64-disk1.img
350 if [[ ! -f ${trusty_image} ]]; then
351     echo "Download image trusty-server-cloudimg-amd64-disk1.img ..."
352     wget -q -nc http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img -P ${image_path}
353 fi
354 sudo cp ${trusty_image} ${DOVETAIL_IMAGES}
355
356 opts="--privileged=true -id"
357
358 docker_volume="-v /var/run/docker.sock:/var/run/docker.sock"
359 dovetail_home_volume="-v ${DOVETAIL_HOME}:${DOVETAIL_HOME}"
360
361 # Pull the image with correct tag
362 DOCKER_REPO='opnfv/dovetail'
363 if [ "$(uname -m)" = 'aarch64' ]; then
364     DOCKER_REPO="${DOCKER_REPO}_$(uname -m)"
365     DOCKER_TAG="latest"
366 fi
367
368 echo "Dovetail: Pulling image ${DOCKER_REPO}:${DOCKER_TAG}"
369 docker pull ${DOCKER_REPO}:$DOCKER_TAG >$redirect
370
371 cmd="docker run ${opts} -e DOVETAIL_HOME=${DOVETAIL_HOME} ${docker_volume} ${dovetail_home_volume} \
372      ${sshkey} ${DOCKER_REPO}:${DOCKER_TAG} /bin/bash"
373 echo "Dovetail: running docker run command: ${cmd}"
374 ${cmd} >${redirect}
375 sleep 5
376 container_id=$(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | awk '{print $1}' | head -1)
377 echo "Container ID=${container_id}"
378 if [ -z ${container_id} ]; then
379     echo "Cannot find ${DOCKER_REPO} container ID ${container_id}. Please check if it is existing."
380     docker ps -a
381     exit 1
382 fi
383 echo "Container Start: docker start ${container_id}"
384 docker start ${container_id}
385 sleep 5
386 docker ps >${redirect}
387 if [ $(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | wc -l) == 0 ]; then
388     echo "The container ${DOCKER_REPO} with ID=${container_id} has not been properly started. Exiting..."
389     exit 1
390 fi
391
392 # Modify tempest_conf.yaml file
393 tempest_conf_file=${DOVETAIL_CONFIG}/tempest_conf.yaml
394 if [[ ${INSTALLER_TYPE} == 'compass' || ${INSTALLER_TYPE} == 'apex' ]]; then
395     volume_device='vdb'
396 else
397     volume_device='vdc'
398 fi
399
400 cat << EOF >$tempest_conf_file
401
402 compute:
403     min_compute_nodes: 2
404     volume_device_name: ${volume_device}
405
406 EOF
407
408 echo "${tempest_conf_file}..."
409 cat ${tempest_conf_file}
410
411 cp_tempest_cmd="docker cp ${DOVETAIL_CONFIG}/tempest_conf.yaml $container_id:/home/opnfv/dovetail/dovetail/userconfig"
412 echo "exec command: ${cp_tempest_cmd}"
413 $cp_tempest_cmd
414
415 if [[ ${TESTSUITE} == 'default' ]]; then
416     testsuite=''
417 else
418     testsuite="--testsuite ${TESTSUITE}"
419 fi
420
421 if [[ ${TESTAREA} == 'mandatory' ]]; then
422     testarea='--mandatory'
423 elif [[ ${TESTAREA} == 'optional' ]]; then
424     testarea="--optional"
425 elif [[ ${TESTAREA} == 'all' ]]; then
426     testarea=""
427 else
428     testarea="--testarea ${TESTAREA}"
429 fi
430
431 run_cmd="dovetail run ${testsuite} ${testarea} --deploy-scenario ${DEPLOY_SCENARIO} -d -r"
432 echo "Container exec command: ${run_cmd}"
433 docker exec $container_id ${run_cmd}
434
435 sudo cp -r ${DOVETAIL_HOME}/results ./
436 result_package=$(find ${DOVETAIL_HOME} -name 'logs_*')
437 echo "Results package is ${result_package}"
438 for item in ${result_package};
439 do
440   sudo mv ${item} ./results
441 done
442
443 # To make sure the file owner is the current user, for the copied results files in the above line
444 echo "Change owner of result files ..."
445 CURRENT_USER=${SUDO_USER:-$USER}
446 PRIMARY_GROUP=$(id -gn $CURRENT_USER)
447 echo "Current user is ${CURRENT_USER}, group is ${PRIMARY_GROUP}"
448 sudo chown -R ${CURRENT_USER}:${PRIMARY_GROUP} ./results
449
450 #remove useless files to save disk space
451 sudo rm -rf ./results/workspace
452 sudo rm -f ./results/yardstick.img
453 sudo rm -f ./results/bottlenecks/tmp*
454
455 echo "Dovetail: done!"
456