Use computes + control for NUM_OS_SYSTEM
[releng.git] / jjb / dovetail / dovetail-run.sh
1 #!/bin/bash
2 ##############################################################################
3 # Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 #the noun INSTALLER is used in community, here is just the example to run.
11 #multi-platforms are supported.
12
13 set -e
14 [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null"
15
16 DEPLOY_TYPE=baremetal
17 [[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt
18
19 DOVETAIL_HOME=${WORKSPACE}/cvp
20 [ -d ${DOVETAIL_HOME} ] && sudo rm -rf ${DOVETAIL_HOME}
21
22 mkdir -p ${DOVETAIL_HOME}
23
24 DOVETAIL_CONFIG=${DOVETAIL_HOME}/pre_config
25 mkdir -p ${DOVETAIL_CONFIG}
26
27 DOVETAIL_IMAGES=${DOVETAIL_HOME}/images
28 mkdir -p ${DOVETAIL_IMAGES}
29
30 ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
31
32 sshkey=""
33 # The path of openrc.sh is defined in fetch_os_creds.sh
34 OPENRC=${DOVETAIL_CONFIG}/env_config.sh
35 CACERT=${DOVETAIL_CONFIG}/os_cacert
36 if [[ ${INSTALLER_TYPE} == 'apex' ]]; then
37     instack_mac=$(sudo virsh domiflist undercloud | grep default | \
38                   grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+")
39     INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'})
40     sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa"
41     if [[ -n $(sudo iptables -L FORWARD |grep "REJECT"|grep "reject-with icmp-port-unreachable") ]]; then
42         #note: this happens only in opnfv-lf-pod1
43         sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
44         sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
45     fi
46 elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then
47     # If production lab then creds may be retrieved dynamically
48     # creds are on the jumphost, always in the same folder
49     sudo cp $LAB_CONFIG/admin-openrc $OPENRC
50     # If dev lab, credentials may not be the default ones, just provide a path to put them into docker
51     # replace the default one by the customized one provided by jenkins config
52 fi
53
54 # Set iptables rule to allow forwarding return traffic for container
55 if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then
56     sudo iptables -I FORWARD -j RETURN
57 fi
58
59 releng_repo=${WORKSPACE}/releng
60 [ -d ${releng_repo} ] && sudo rm -rf ${releng_repo}
61 git clone https://gerrit.opnfv.org/gerrit/releng ${releng_repo} >/dev/null
62
63 pharos_repo=${WORKSPACE}/pharos
64 [ -d ${pharos_repo} ] && sudo rm -rf ${pharos_repo}
65 git clone https://git.opnfv.org/pharos ${pharos_repo} >/dev/null
66
67 if [[ ${INSTALLER_TYPE} != 'joid' ]]; then
68     echo "SUT branch is $SUT_BRANCH"
69     echo "dovetail branch is $BRANCH"
70     BRANCH_BACKUP=$BRANCH
71     export BRANCH=$SUT_BRANCH
72     ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${CACERT} >${redirect}
73     export BRANCH=$BRANCH_BACKUP
74 fi
75
76 if [[ -f $OPENRC ]]; then
77     echo "INFO: openstack credentials path is $OPENRC"
78     if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then
79         if [[ -f ${CACERT} ]]; then
80             echo "INFO: ${INSTALLER_TYPE} openstack cacert file is ${CACERT}"
81             echo "export OS_CACERT=${CACERT}" >> ${OPENRC}
82         else
83             echo "ERROR: Can't find ${INSTALLER_TYPE} openstack cacert file. Please check if it is existing."
84             sudo ls -al ${DOVETAIL_CONFIG}
85             exit 1
86         fi
87     fi
88     echo "export EXTERNAL_NETWORK=${EXTERNAL_NETWORK}" >> ${OPENRC}
89 else
90     echo "ERROR: cannot find file $OPENRC. Please check if it is existing."
91     sudo ls -al ${DOVETAIL_CONFIG}
92     exit 1
93 fi
94
95 if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "fuel" ]]; then
96     sed -i "s#/etc/ssl/certs/mcp_os_cacert#${CACERT}#g" ${OPENRC}
97 fi
98 cat $OPENRC
99
100 # These packages are used for parsing yaml files and decrypting ipmi user and password.
101 sudo pip install shyaml
102 sudo yum install -y rubygems || sudo apt-get install -y ruby
103 sudo gem install hiera-eyaml
104
105 if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then
106     compass_repo=${WORKSPACE}/compass4nfv/
107     git clone https://github.com/opnfv/compass4nfv.git ${compass_repo} >/dev/null
108     scenario_file=${compass_repo}/deploy/conf/hardware_environment/$NODE_NAME/os-nosdn-nofeature-ha.yml
109     ipmiIp=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiIp)
110     ipmiPass=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiPass)
111     ipmiUser=root
112     jumpserver_ip=$(ifconfig | grep -A 5 docker0 | grep "inet addr" | cut -d ':' -f 2 | cut -d ' ' -f 1)
113
114     cat << EOF >${DOVETAIL_CONFIG}/pod.yaml
115 nodes:
116 - {ip: ${jumpserver_ip}, name: node0, password: root, role: Jumpserver, user: root}
117 - {ip: 10.1.0.50, name: node1, password: root, role: controller, user: root,
118    ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}}
119 - {ip: 10.1.0.51, name: node2, password: root, role: controller, user: root}
120 - {ip: 10.1.0.52, name: node3, password: root, role: controller, user: root}
121 - {ip: 10.1.0.53, name: node4, password: root, role: compute, user: root}
122 - {ip: 10.1.0.54, name: node5, password: root, role: compute, user: root}
123
124 EOF
125 fi
126
127 if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then
128     fuel_ctl_ssh_options="${ssh_options} -i ${SSH_KEY}"
129     ssh_user="ubuntu"
130     fuel_ctl_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
131             "sudo salt 'cfg*' pillar.get _param:openstack_control_address --out text| \
132                 cut -f2 -d' '")
133     fuel_cmp_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
134             "sudo salt 'cmp001*' pillar.get _param:openstack_control_address --out text| \
135                 cut -f2 -d' '")
136     fuel_dbs_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
137             "sudo salt 'dbs01*' pillar.get _param:openstack_control_address --out text| \
138                 cut -f2 -d' '")
139     fuel_msg_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
140             "sudo salt 'msg01*' pillar.get _param:openstack_control_address --out text| \
141                 cut -f2 -d' '")
142     ipmi_index=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \
143             "sudo salt 'ctl*' network.ip_addrs cidr=${fuel_ctl_ip} --out text | grep ${fuel_ctl_ip} | cut -c 5")
144
145     organization="$(cut -d'-' -f1 <<< "${NODE_NAME}")"
146     pod_name="$(cut -d'-' -f2 <<< "${NODE_NAME}")"
147     pdf_file=${pharos_repo}/labs/${organization}/${pod_name}.yaml
148     ipmiIp=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.address)
149     ipmiIp="$(cut -d'/' -f1 <<< "${ipmiIp}")"
150     ipmiPass=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.pass)
151     ipmiUser=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.user)
152     [[ $ipmiUser == ENC* ]] && ipmiUser=$(eyaml decrypt -s ${ipmiUser//[[:blank:]]/})
153     [[ $ipmiPass == ENC* ]] && ipmiPass=$(eyaml decrypt -s ${ipmiPass//[[:blank:]]/})
154
155     cat << EOF >${DOVETAIL_CONFIG}/pod.yaml
156 nodes:
157 - {ip: ${INSTALLER_IP}, name: node0, key_filename: /home/opnfv/userconfig/pre_config/id_rsa,
158    role: Jumpserver, user: ${ssh_user}}
159 - {ip: ${fuel_ctl_ip}, name: node1, key_filename: /home/opnfv/userconfig/pre_config/id_rsa,
160    role: controller, user: ${ssh_user}, ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}}
161 - {ip: ${fuel_msg_ip}, name: msg01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}}
162 - {ip: ${fuel_cmp_ip}, name: cmp01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}}
163 - {ip: ${fuel_dbs_ip}, name: dbs01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}}
164 EOF
165 fi
166
167 if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then
168     set +e
169
170     sudo pip install virtualenv
171
172     cd ${releng_repo}/modules
173     sudo virtualenv venv
174     source venv/bin/activate
175     sudo pip install -e ./ >/dev/null
176     sudo pip install netaddr
177
178     if [[ ${INSTALLER_TYPE} == compass ]]; then
179         options="-u root -p root"
180     elif [[ ${INSTALLER_TYPE} == fuel ]]; then
181         options="-u root -p r00tme"
182     elif [[ ${INSTALLER_TYPE} == apex ]]; then
183         options="-u stack -k /root/.ssh/id_rsa"
184     elif [[ ${INSTALLER_TYPE} == daisy ]]; then
185         options="-u root -p r00tme"
186     else
187         echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently."
188         echo "HA test cases may not run properly."
189     fi
190
191     cmd="sudo python ${releng_repo}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \
192          -i ${INSTALLER_IP} ${options} -f ${DOVETAIL_CONFIG}/pod.yaml \
193          -s /home/opnfv/userconfig/pre_config/id_rsa"
194     echo ${cmd}
195     ${cmd}
196
197     deactivate
198
199     set -e
200
201     cd ${WORKSPACE}
202 fi
203
204 if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then
205     sudo chmod 666 ${DOVETAIL_CONFIG}/pod.yaml
206     echo "Adapt process info for $INSTALLER_TYPE ..."
207     if [ "$INSTALLER_TYPE" == "apex" ]; then
208         cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml
209 process_info:
210 - {testcase_name: dovetail.ha.rabbitmq, attack_process: rabbitmq_server}
211 EOF
212     elif [ "$INSTALLER_TYPE" == "fuel" ]; then
213         cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml
214 process_info:
215 - {testcase_name: dovetail.ha.cinder_api, attack_process: cinder_wsgi}
216 - {testcase_name: dovetail.ha.rabbitmq, attack_process: rabbitmq-server, attack_host: msg01}
217 - {testcase_name: dovetail.ha.neutron_l3_agent, attack_process: neutron-l3-agent, attack_host: cmp01}
218 - {testcase_name: dovetail.ha.database, attack_process: mysqld, attack_host: dbs01}
219 EOF
220     fi
221
222     echo "file ${DOVETAIL_CONFIG}/pod.yaml:"
223     cat ${DOVETAIL_CONFIG}/pod.yaml
224 else
225     echo "Error: cannot find file ${DOVETAIL_CONFIG}/pod.yaml. Please check if it is existing."
226     sudo ls -al ${DOVETAIL_CONFIG}
227     echo "HA test cases may not run properly."
228 fi
229
230 if [ "$INSTALLER_TYPE" == "fuel" ]; then
231     if [[ "${SUT_BRANCH}" =~ "danube" ]]; then
232         echo "Fetching id_rsa file from jump_server $INSTALLER_IP..."
233         sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa
234     else
235         cp ${SSH_KEY} ${DOVETAIL_CONFIG}/id_rsa
236     fi
237 fi
238
239 if [ "$INSTALLER_TYPE" == "apex" ]; then
240     echo "Fetching id_rsa file from jump_server $INSTALLER_IP..."
241     sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa
242 fi
243
244 if [ "$INSTALLER_TYPE" == "daisy" ]; then
245     echo "Fetching id_dsa file from jump_server $INSTALLER_IP..."
246     sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_dsa ${DOVETAIL_CONFIG}/id_rsa
247 fi
248
249
250 image_path=${HOME}/opnfv/dovetail/images
251 if [[ ! -d ${image_path} ]]; then
252     mkdir -p ${image_path}
253 fi
254 # sdnvpn test case needs to download this image first before running
255 ubuntu_image=${image_path}/ubuntu-16.04-server-cloudimg-amd64-disk1.img
256 if [[ ! -f ${ubuntu_image} ]]; then
257     echo "Download image ubuntu-16.04-server-cloudimg-amd64-disk1.img ..."
258     wget -q -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${image_path}
259 fi
260 sudo cp ${ubuntu_image} ${DOVETAIL_IMAGES}
261
262 # yardstick and bottlenecks need to download this image first before running
263 cirros_image=${image_path}/cirros-0.3.5-x86_64-disk.img
264 if [[ ! -f ${cirros_image} ]]; then
265     echo "Download image cirros-0.3.5-x86_64-disk.img ..."
266     wget -q -nc http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img -P ${image_path}
267 fi
268 sudo cp ${cirros_image} ${DOVETAIL_IMAGES}
269
270 # functest needs to download this image first before running
271 cirros_image=${image_path}/cirros-0.4.0-x86_64-disk.img
272 if [[ ! -f ${cirros_image} ]]; then
273     echo "Download image cirros-0.4.0-x86_64-disk.img ..."
274     wget -q -nc http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img -P ${image_path}
275 fi
276 sudo cp ${cirros_image} ${DOVETAIL_IMAGES}
277
278 # snaps_smoke test case needs to download this image first before running
279 ubuntu14_image=${image_path}/ubuntu-14.04-server-cloudimg-amd64-disk1.img
280 if [[ ! -f ${ubuntu14_image} ]]; then
281     echo "Download image ubuntu-14.04-server-cloudimg-amd64-disk1.img ..."
282     wget -q -nc https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img -P ${image_path}
283 fi
284 sudo cp ${ubuntu14_image} ${DOVETAIL_IMAGES}
285
286 # cloudify_ims test case needs to download these 2 images first before running
287 cloudify_image=${image_path}/cloudify-manager-premium-4.0.1.qcow2
288 if [[ ! -f ${cloudify_image} ]]; then
289     echo "Download image cloudify-manager-premium-4.0.1.qcow2 ..."
290     wget -q -nc http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 -P ${image_path}
291 fi
292 sudo cp ${cloudify_image} ${DOVETAIL_IMAGES}
293 trusty_image=${image_path}/trusty-server-cloudimg-amd64-disk1.img
294 if [[ ! -f ${trusty_image} ]]; then
295     echo "Download image trusty-server-cloudimg-amd64-disk1.img ..."
296     wget -q -nc http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img -P ${image_path}
297 fi
298 sudo cp ${trusty_image} ${DOVETAIL_IMAGES}
299
300 opts="--privileged=true -id"
301
302 docker_volume="-v /var/run/docker.sock:/var/run/docker.sock"
303 dovetail_home_volume="-v ${DOVETAIL_HOME}:${DOVETAIL_HOME}"
304
305 # Pull the image with correct tag
306 DOCKER_REPO='opnfv/dovetail'
307 if [ "$(uname -m)" = 'aarch64' ]; then
308     DOCKER_REPO="${DOCKER_REPO}_$(uname -m)"
309     DOCKER_TAG="latest"
310 fi
311
312 echo "Dovetail: Pulling image ${DOCKER_REPO}:${DOCKER_TAG}"
313 docker pull ${DOCKER_REPO}:$DOCKER_TAG >$redirect
314
315 cmd="docker run ${opts} -e DOVETAIL_HOME=${DOVETAIL_HOME} ${docker_volume} ${dovetail_home_volume} \
316      ${sshkey} ${DOCKER_REPO}:${DOCKER_TAG} /bin/bash"
317 echo "Dovetail: running docker run command: ${cmd}"
318 ${cmd} >${redirect}
319 sleep 5
320 container_id=$(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | awk '{print $1}' | head -1)
321 echo "Container ID=${container_id}"
322 if [ -z ${container_id} ]; then
323     echo "Cannot find ${DOCKER_REPO} container ID ${container_id}. Please check if it is existing."
324     docker ps -a
325     exit 1
326 fi
327 echo "Container Start: docker start ${container_id}"
328 docker start ${container_id}
329 sleep 5
330 docker ps >${redirect}
331 if [ $(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | wc -l) == 0 ]; then
332     echo "The container ${DOCKER_REPO} with ID=${container_id} has not been properly started. Exiting..."
333     exit 1
334 fi
335
336 # Modify tempest_conf.yaml file
337 tempest_conf_file=${DOVETAIL_CONFIG}/tempest_conf.yaml
338 if [[ ${INSTALLER_TYPE} == 'compass' || ${INSTALLER_TYPE} == 'apex' ]]; then
339     volume_device='vdb'
340 else
341     volume_device='vdc'
342 fi
343
344 cat << EOF >$tempest_conf_file
345
346 compute:
347     min_compute_nodes: 2
348     volume_device_name: ${volume_device}
349
350 EOF
351
352 echo "${tempest_conf_file}..."
353 cat ${tempest_conf_file}
354
355 cp_tempest_cmd="docker cp ${DOVETAIL_CONFIG}/tempest_conf.yaml $container_id:/home/opnfv/dovetail/dovetail/userconfig"
356 echo "exec command: ${cp_tempest_cmd}"
357 $cp_tempest_cmd
358
359 if [[ ${TESTSUITE} == 'default' ]]; then
360     testsuite=''
361 else
362     testsuite="--testsuite ${TESTSUITE}"
363 fi
364
365 if [[ ${TESTAREA} == 'mandatory' ]]; then
366     testarea='--mandatory'
367 elif [[ ${TESTAREA} == 'optional' ]]; then
368     testarea="--optional"
369 elif [[ ${TESTAREA} == 'all' ]]; then
370     testarea=""
371 else
372     testarea="--testarea ${TESTAREA}"
373 fi
374
375 run_cmd="dovetail run ${testsuite} ${testarea} -d"
376 echo "Container exec command: ${run_cmd}"
377 docker exec $container_id ${run_cmd}
378
379 sudo cp -r ${DOVETAIL_HOME}/results ./
380 # To make sure the file owner is the current user, for the copied results files in the above line
381 echo "Change owner of result files ..."
382 CURRENT_USER=${SUDO_USER:-$USER}
383 PRIMARY_GROUP=$(id -gn $CURRENT_USER)
384 echo "Current user is ${CURRENT_USER}, group is ${PRIMARY_GROUP}"
385 sudo chown -R ${CURRENT_USER}:${PRIMARY_GROUP} ./results
386
387 #remove useless files to save disk space
388 sudo rm -rf ./results/workspace
389 sudo rm -f ./results/yardstick.img
390 sudo rm -f ./results/tmp*
391
392 echo "Dovetail: done!"
393