2 # http://www.apache.org/licenses/LICENSE-2.0 *
4 # Unless required by applicable law or agreed to in writing, *
5 # software distributed under the License is distributed on an *
6 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
7 # KIND, either express or implied. See the License for the *
8 # specific language governing permissions and limitations *
11 # This script checks if deployments are working or and then
12 # starts the specified containers in case one of the containers
13 # crash. The only solution is restarting docker as of now.
18 modules=(testapi reporting)
20 ## Ports of the modules
21 declare -A ports=( ["testapi"]="8082" ["reporting"]="8084")
23 ## Urls to check if the modules are deployed or not ?
24 declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \
25 ["reporting"]="http://testresults.opnfv.org/reporting/index.html")
27 ### Functions related to checking.
29 function is_deploying() {
30 xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1")
31 building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml")
32 if [[ $building == "false" ]]
40 function get_docker_status() {
41 status=$(service docker status | sed -n 3p | cut -d ' ' -f5)
42 echo -e "Docker status: $status"
43 if [ $status = "active" ]
51 function check_connectivity() {
52 echo "Checking $1 connection : $2"
53 cmd=`curl --head -m10 --request GET ${2} | grep '200 OK' > /dev/null`
55 if [[ $rc == 0 ]]; then
62 function check_modules() {
63 echo -e "Checking modules"
65 for module in "${modules[@]}"
67 if is_deploying $module; then
70 if ! check_connectivity $module "${urls[$module]}"; then
71 echo -e "$module failed"
72 failed_modules+=($module)
75 if [ ! -z "$failed_modules" ]; then
76 echo -e "Failed Modules: $failed_modules"
79 echo -e "All modules working good"
84 ### Functions related fixes.
86 function restart_docker_fix() {
87 echo -e "Running restart_docker_fix"
88 service docker restart
89 start_containers_fix "${modules[@]}"
92 function docker_proxy_fix() {
93 echo -e "Running docker_proxy_fix"
95 for module in "${fix_modules[@]}"
97 echo -e "Kill docker proxy and restart containers"
98 pid=$(netstat -nlp | grep :${ports[$module]} | awk '{print $7}' | cut -d'/' -f1)
100 if [ ! -z "$pid" ]; then
102 start_container_fix $module
107 function start_containers_fix() {
108 start_modules=("${@}")
109 for module in "${start_modules[@]}"
111 start_container_fix $module
115 function start_container_fix() {
116 echo -e "Starting a container $module"
117 sudo docker stop $module
118 sudo docker start $module
120 if ! check_connectivity $module "${urls[$module]}"; then
121 echo -e "Starting an old container $module_old"
122 sudo docker stop $module
123 sudo docker start $module"_old"
131 echo -e "WatchDog Started"
133 echo -e `date "+%Y-%m-%d %H:%M:%S.%N"`
136 ## If the problem is related to docker daemon
138 if ! get_docker_status; then
140 if ! check_modules; then
141 echo -e "Watchdog failed while restart_docker_fix"
146 ## If the problem is related to docker proxy
148 if ! check_modules; then
149 docker_proxy_fix "${failed_modules[@]}"
152 ## If any other problem : restart docker
154 if ! check_modules; then
158 ## If nothing works out
160 if ! check_modules; then
161 echo -e "Watchdog failed"