2 ##############################################################################
3 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ##############################################################################
12 ## run_rubbos_interlnal.sh is executed in rubbos-controller
13 ## Usage: run_rubbos_internal.sh rubbos.conf local_result_dir
15 echo "Need Two argument!"
20 controller_host=`hostname`
22 #REMOTE_GIT_REPO=git://git.opnfv.org/bottlenecks
23 REMOTE_GIT_REPO=https://gerrit.opnfv.org/gerrit/bottlenecks
24 REMOTE_ARTIFACTS_REPO=http://artifacts.opnfv.org/bottlenecks/rubbos/rubbos_files.tar.gz
25 LOCAL_GIT_REPO=/home/ubuntu/btnks-git
26 LOCAL_ARTIFACTS_REPO=/home/ubuntu/btnks-artifacts
27 LOCAL_RUBBOS_MANIFESTS_HOME=/home/ubuntu/btnks-git/bottlenecks/testsuites/rubbos/puppet_manifests
28 LOCAL_ARTIFACTS_RUBBOS_HOME=/home/ubuntu/btnks-artifacts/rubbos_files
30 SSH_ARGS="-o StrictHostKeyChecking=no -o BatchMode=yes -i /home/ubuntu/.ssh/id_rsa"
32 # conf properties from the input config file
46 # conf info used in this script
50 remote_client_servers=
59 # Other variables used in this script
65 if [ ${#line} -gt 0 ] && [ ${line:0:1} != "#" ] && [ ${line:0:1} != "[" ];then
69 e_arr=(${line[1]//:/ })
70 controller_host=${e_arr[0]}
71 controller_ip=${e_arr[2]}
72 hostname_arr=("${hostname_arr[@]}" "${e_arr[0]}")
73 hostip_arr=("${hostip_arr[@]}" "${e_arr[2]}");;
75 elems=(${line[1]//,/ })
76 for e in "${elems[@]}";do
78 client_servers=${client_servers}${e_arr[0]}","
79 hostname_arr=("${hostname_arr[@]}" "${e_arr[0]}")
80 hostip_arr=("${hostip_arr[@]}" "${e_arr[1]}")
82 client_servers=${client_servers%,};;
84 e_arr=(${line[1]//:/ })
85 web_servers=${web_servers}${e_arr[0]}","
86 hostname_arr=("${hostname_arr[@]}" "${e_arr[0]}")
87 hostip_arr=("${hostip_arr[@]}" "${e_arr[1]}")
88 web_servers=${e_arr[0]};;
90 elems=(${line[1]//,/ })
91 for e in "${elems[@]}";do
93 app_servers=${app_servers}${e_arr[0]}","
94 hostname_arr=("${hostname_arr[@]}" "${e_arr[0]}")
95 hostip_arr=("${hostip_arr[@]}" "${e_arr[1]}")
97 app_servers=${app_servers%,};;
99 if [ "x"${line[1]} != "x" ]; then
100 e_arr=(${line[1]//:/ })
101 hostname_arr=("${hostname_arr[@]}" "${e_arr[0]}")
102 hostip_arr=("${hostip_arr[@]}" "${e_arr[1]}")
103 cjdbc_controller=${e_arr[0]}
107 elems=(${line[1]//,/ })
108 for e in "${elems[@]}";do
110 database_servers=${database_servers}${e_arr[0]}","
111 hostname_arr=("${hostname_arr[@]}" "${e_arr[0]}")
112 hostip_arr=("${hostip_arr[@]}" "${e_arr[1]}")
114 database_servers=${database_servers%,};;
116 database_port=${line[1]};;
118 db_username=${line[1]};;
120 db_password=${line[1]};;
122 clients_per_node=${line[@]:1:${#line[@]-1}};;
124 rubbos_app=${line[1]};;
126 rubbos_app_tools=${line[1]};;
128 rubbos_home=${line[1]};;
131 done < $local_cfg_path
133 clients_arr=(${client_servers//,/ })
134 clients_per_node_arr=(${clients_per_node})
135 webservers_arr=(${web_servers//,/ })
136 appservers_arr=(${app_servers//,/ })
137 dbservers_arr=(${database_servers//,/ })
138 all_agents_arr=("${clients_arr[@]}" "${webservers_arr[@]}" "${appservers_arr[@]}" "${dbservers_arr[@]}")
139 bench_client=${clients_arr[0]}
140 len=${#clients_arr[@]}
141 if [ $len -gt 1 ]; then
142 remote_clients_arr=(${clients_arr[@]:1:$len-1})
143 remote_client_servers=${client_servers#*,}
146 echo "-------------Main conf info:----------"
148 while [ $i -lt ${#hostname_arr[@]} ]; do
149 echo ${hostip_arr[$i]}" "${hostname_arr[$i]}
152 echo "clients_arr: "${clients_arr[@]}
153 echo "bench_client: "$bench_client
154 echo "remote_client_servers: "$remote_client_servers
155 echo "remote_clients_arr: "${remote_clients_arr[@]}
156 echo "clients_per_node_arr: "${clients_per_node_arr[@]}
157 echo "webservers_arr: "${webservers_arr[@]}
158 echo "appservers_arr: "${appservers_arr[@]}
159 echo "dbservers_arr: "${dbservers_arr[@]}
160 echo "all agents: "${all_agents_arr[@]}
163 fetch_remote_resources() {
164 if [ -d $LOCAL_GIT_REPO ];then
165 rm -rf $LOCAL_GIT_REPO
167 mkdir -p $LOCAL_GIT_REPO
168 #sudo dpkg --configure -a
170 sudo apt-get install -y git
172 git clone ${REMOTE_GIT_REPO}
174 if [ -d $LOCAL_ARTIFACTS_REPO ];then
175 rm -rf $LOCAL_ARTIFACTS_REPO
177 mkdir -p $LOCAL_ARTIFACTS_REPO
178 cd $LOCAL_ARTIFACTS_REPO
179 wget -nv ${REMOTE_ARTIFACTS_REPO}
180 tar xvzf rubbos_files.tar.gz
182 if [ -d $local_results_dir ];then
183 rm -rf $local_results_dir
185 mkdir -p $local_results_dir
188 # ssh all vms/instances once only after first creation
190 echo "127.0.0.1 $(hostname)" >> /etc/hosts
191 echo "write hosts file: 127.0.0.1 $(hostname)"
192 cp ${LOCAL_GIT_REPO}/bottlenecks/utils/infra_setup/bottlenecks_key/bottlenecks_key /home/ubuntu/.ssh/id_rsa
193 sudo chmod 0600 /home/ubuntu/.ssh/id_rsa
194 echo 'StrictHostKeyChecking no' > /home/ubuntu/.ssh/config
195 sudo chown -R ubuntu:ubuntu /home/ubuntu/.ssh
197 while [ $i -lt ${#hostip_arr[@]} ]; do
198 echo ${hostip_arr[$i]}" "${hostname_arr[$i]} >> /etc/hosts
201 echo "Done controller."
204 while [ $i -lt ${#hostip_arr[@]} ]; do
205 if [ ${hostname_arr[$i]} == ${controller_host} ];then
209 echo "Processing: "${hostip_arr[$i]}" "${hostname_arr[$i]}
210 echo "ssh *sudo hostname* test:"
211 ssh ${SSH_ARGS} ubuntu@${hostip_arr[$i]} "sudo hostname"
213 ssh ${SSH_ARGS} ubuntu@${hostip_arr[$i]} "sudo cp /etc/hosts /home/ubuntu/ && sudo chmod 646 /home/ubuntu/hosts"
214 ssh ${SSH_ARGS} ubuntu@${hostip_arr[$i]} "echo 127.0.0.1 ${hostname_arr[$i]} >> /home/ubuntu/hosts"
216 while [ $j -lt ${#hostip_arr[@]} ];do
217 local host_item=${hostip_arr[$j]}" "${hostname_arr[$j]}
218 ssh ${SSH_ARGS} ubuntu@${hostip_arr[$i]} "echo ${host_item} >> /home/ubuntu/hosts"
221 ssh ${SSH_ARGS} ubuntu@${hostip_arr[$i]} "sudo chmod 644 /home/ubuntu/hosts && sudo cp /home/ubuntu/hosts /etc/ && sudo rm -rf /home/ubuntu/hosts"
224 sudo ssh ${SSH_ARGS} ubuntu@${hostip_arr[$i]} "echo 'StrictHostKeyChecking no' > /home/ubuntu/.ssh/config"
225 sudo scp ${SSH_ARGS} /home/ubuntu/.ssh/id_rsa ubuntu@${hostip_arr[$i]}:/home/ubuntu/.ssh/
231 start_puppet_service() {
233 sudo service puppetserver stop
234 sudo service puppetserver start
235 sudo service puppetserver status
236 # Start all puppet agents
237 for host in "${all_agents_arr[@]}";do
238 echo "start puppet agent on:"${host}
239 ssh ${SSH_ARGS} ubuntu@${host} "sudo service puppet status"
240 ssh ${SSH_ARGS} ubuntu@${host} "sudo service puppet stop"
241 ssh ${SSH_ARGS} ubuntu@${host} "sudo service puppet start --no-client"
242 ssh ${SSH_ARGS} ubuntu@${host} "sudo service puppet status"
245 sudo puppet cert list --all
246 sudo puppet cert sign --all
247 sudo puppet cert list --all
251 # It requires one local file path which needs to be replaced
253 echo "_replace file: "$1
254 sed -i 's#REPLACED_RUBBOS_APP_TOOLS#'${rubbos_app_tools}'#g' $1
255 sed -i 's#REPLACED_RUBBOS_APP#'${rubbos_app}'#g' $1
256 sed -i 's#REPLACED_RUBBOS_HOME#'${rubbos_home}'#g' $1
257 local mysql_jdbc_url="jdbc:mysql://"${database_servers}":"${database_port}"/rubbos"
258 sed -i 's#REPLACED_MYSQL_JDBC_DB_URL#'${mysql_jdbc_url}'#g' $1
259 sed -i 's/REPLACED_MYSQL_USERNAME/'${db_username}'/g' $1
260 sed -i 's/REPLACED_MYSQL_PASSWORD/'${db_password}'/g' $1
264 while [ $i -lt ${#appservers_arr} ];do
266 handlers=${handlers}${handler_name}","
267 handler_details=${handler_details}"worker."${handler_name}".port=8009\n"
268 handler_details=${handler_details}"worker."${handler_name}".host="${appservers_arr[$i]}"\n"
269 handler_details=${handler_details}"worker."${handler_name}".type=ajp13\n"
270 handler_details=${handler_details}"worker."${handler_name}".lbfactor=1\n"
273 handlers=${handlers%,}
274 sed -i 's/REPLACED_HANDLERS_DETAILS/'${handler_details}'/g' $1
275 sed -i 's/REPLACED_HANDLERS/'${handlers}'/g' $1
277 sed -i 's/REPLACED_WEB_SERVER/'${web_servers}'/g' $1
278 sed -i 's/REPLACED_APPLICATION_SERVER/'${app_servers}'/g' $1
279 sed -i 's/REPLACED_DB_SERVER/'${database_servers}'/g' $1
280 sed -i 's/REPLACED_CLIENT_SERVERS/'${remote_client_servers}'/g' $1
284 # it requires one input string
285 _to_puppet_class_nodes() {
286 echo "_to_puppet_class_nodes "$1
288 nodes_arr=(${1//,/ })
290 while [ $i -lt ${#nodes_arr[@]} ];do
291 class_nodes=${class_nodes}"'"${nodes_arr[$i]}"',"
294 class_nodes=${class_nodes%,}
299 for host in "${clients_arr[@]}"; do
300 ssh ${SSH_ARGS} ubuntu@${host} 'sudo puppet agent -t' &
302 for host in "${webservers_arr[@]}"; do
303 ssh ${SSH_ARGS} ubuntu@${host} 'sudo puppet agent -t'
305 for host in "${appservers_arr[@]}"; do
306 ssh ${SSH_ARGS} ubuntu@${host} 'sudo puppet agent -t'
308 for host in "${dbservers_arr[@]}"; do
309 ssh ${SSH_ARGS} ubuntu@${host} 'sudo puppet agent -t'
313 prepare_manifests() {
315 sudo cp -r ${LOCAL_RUBBOS_MANIFESTS_HOME}/modules/* /etc/puppet/modules/
318 sudo cp -r ${LOCAL_ARTIFACTS_RUBBOS_HOME}/modules/* /etc/puppet/modules/
320 # adjust corresponding configuration files (pre-catalog)
321 _replace_text /etc/puppet/modules/rubbos_tomcat/files/tomcat_sl/build.properties
322 _replace_text /etc/puppet/modules/rubbos_tomcat/files/tomcat_sl/Config.java
323 _replace_text /etc/puppet/modules/rubbos_tomcat/files/tomcat_sl/mysql.properties
324 _replace_text /etc/puppet/modules/rubbos_httpd/files/apache_conf/workers.properties
325 _replace_text /etc/puppet/modules/rubbos_httpd/files/apache_conf/httpd.conf
326 _replace_text /etc/puppet/modules/rubbos_client/files/build.properties
327 _replace_text /etc/puppet/modules/rubbos_client/files/rubbos.properties.template
328 _replace_text /etc/puppet/modules/rubbos_client/files/run_emulator.sh
332 # start all (exec catalog)
333 if [ "x"$1 == "xstart" ];then
334 echo "--> Start to execute catalogs in all agents..."
335 sudo cp ${LOCAL_RUBBOS_MANIFESTS_HOME}/site_on.pp /etc/puppet/manifests/site.pp
337 echo "--> Finish to execute catalogs in all agents."
338 elif [ "x"$1 == "xclean" ];then
339 echo "--> Cleanup all agents..."
340 sudo cp ${LOCAL_RUBBOS_MANIFESTS_HOME}/site_off.pp /etc/puppet/manifests/site.pp
342 echo "--> Finish to cleanup all agents."
347 # prepare data in db servers
348 for host in "${dbservers_arr[@]}"; do
349 ssh ${SSH_ARGS} ubuntu@${host} 'sudo scp ubuntu@'${controller_host}':/etc/puppet/modules/rubbos_mysql/files/rubbos_data_sql.tar.gz '${rubbos_home}''
350 ssh ${SSH_ARGS} ubuntu@${host} 'cd '${rubbos_home}' && sudo ./prepare_rubbos_mysql_db.sh ./rubbos_data_sql.tar.gz ./rubbos_data_sql_dir'
353 # run emulator.sh ( Modify rubbos.properties file first)
354 ssh ${SSH_ARGS} ubuntu@${bench_client} 'sudo rm -rf '${rubbos_home}'/bench/bench'
355 for x in "${clients_per_node_arr[@]}";do
356 echo "run emulator with clients_per_node="$x
357 for host in "${clients_arr[@]}";do
358 ssh ${SSH_ARGS} ubuntu@${host} "sed -e 's/REPLACED_NUMBER_OF_CLIENTS_PER_NODE/'${x}'/g' '${rubbos_home}'/Client/rubbos.properties.template > '${rubbos_home}'/Client/rubbos.properties "
360 ssh ${SSH_ARGS} ubuntu@${bench_client} 'cd '${rubbos_home}'/bench && ./run_emulator.sh'
365 # collect results, from bench_host to controller
366 scp ${SSH_ARGS} -r ubuntu@${bench_client}:${rubbos_home}/bench/bench/* ${local_results_dir}
370 # post-process results and push to the database of dashboard
371 python ${LOCAL_GIT_REPO}/bottlenecks/utils/dashboard/process_data.py -i ${local_results_dir} \
372 -c ${LOCAL_GIT_REPO}/bottlenecks/utils/dashboard/dashboard.yaml \
374 -o ${local_results_dir}/rubbos.out \
379 echo "==> read_conf:"
381 echo "==> fetch_remote_resources:"
382 fetch_remote_resources
383 echo "==> direct_ssh:"
385 echo "==> start_puppet_service:"
387 echo "==> prepare_manifests:"
389 echo "==> execute_catalog start:"
390 execute_catalog start
391 echo "==> run_emulator:"
393 echo "==> collect_results (to controller:${local_results_dir}):"
395 echo "==> process_results:"
397 echo "==> execute_catalog clean:"
398 execute_catalog clean