2 # http://www.apache.org/licenses/LICENSE-2.0 *
4 # Unless required by applicable law or agreed to in writing, *
5 # software distributed under the License is distributed on an *
6 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
7 # KIND, either express or implied. See the License for the *
8 # specific language governing permissions and limitations *
11 # This script checks if deployments are working or and then
12 # starts the specified containers in case one of the containers
13 # crash. The only solution is restarting docker as of now.
18 modules=(testapi reporting)
20 ## Ports of the modules
21 declare -A ports=( ["testapi"]="8082" ["reporting"]="8084")
23 ## Urls to check if the modules are deployed or not ?
24 declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \
25 ["reporting"]="http://testresults.opnfv.org/reporting/index.html")
27 ### Functions related to checking.
29 function is_deploying() {
30 xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-docker-deploy-master/lastBuild/api/xml?depth=1")
31 building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml")
32 if [[ $building == "false" ]]
40 function get_docker_status() {
41 status=$(service docker status | sed -n 3p | cut -d ' ' -f5)
42 echo -e "Docker status: $status"
43 if [ $status = "active" ]
51 function check_connectivity() {
52 echo "Checking $1 connection : $2"
53 cmd=`curl --head -m10 --request GET ${2} | grep '200 OK' > /dev/null`
55 if [[ $rc == 0 ]]; then
62 function check_modules() {
63 echo -e "Checking modules"
65 for module in "${modules[@]}"
67 if ! check_connectivity $module "${urls[$module]}"; then
68 if ! is_deploying $module; then
69 echo -e "$module failed"
70 failed_modules+=($module)
74 if [ ! -z "$failed_modules" ]; then
75 echo -e "Failed Modules: $failed_modules"
78 echo -e "All modules working good"
83 ### Functions related fixes.
85 function restart_docker_fix() {
86 echo -e "Running restart_docker_fix"
87 service docker restart
88 start_containers_fix "${modules[@]}"
91 function docker_proxy_fix() {
92 echo -e "Running docker_proxy_fix"
94 for module in "${fix_modules[@]}"
96 echo -e "Kill docker proxy and restart containers"
97 pid=$(netstat -nlp | grep :${ports[$module]} | awk '{print $7}' | cut -d'/' -f1)
99 if [ ! -z "$pid" ]; then
101 start_container_fix $module
106 function start_containers_fix() {
107 start_modules=("${@}")
108 for module in "${start_modules[@]}"
110 start_container_fix $module
114 function start_container_fix() {
115 echo -e "Starting a container $module"
116 sudo docker restart $module
118 if ! check_connectivity $module "${urls[$module]}"; then
119 echo -e "Starting an old container $module_old"
120 sudo docker restart $module"_old"
128 echo -e "WatchDog Started"
130 echo -e `date "+%Y-%m-%d %H:%M:%S.%N"`
133 ## If the problem is related to docker daemon
135 if ! get_docker_status; then
137 if ! check_modules; then
138 echo -e "Watchdog failed while restart_docker_fix"
143 ## If the problem is related to docker proxy
145 if ! check_modules; then
146 docker_proxy_fix "${failed_modules[@]}"
149 ## If any other problem : restart docker
151 if ! check_modules; then
155 ## If nothing works out
157 if ! check_modules; then
158 echo -e "Watchdog failed"