2 # http://www.apache.org/licenses/LICENSE-2.0 *
4 # Unless required by applicable law or agreed to in writing, *
5 # software distributed under the License is distributed on an *
6 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
7 # KIND, either express or implied. See the License for the *
8 # specific language governing permissions and limitations *
11 # This script checks if deployments are working or and then
12 # starts the specified containers in case one of the containers
13 # crash. The only solution is restarting docker as of now.
18 modules=(testapi reporting)
20 ## Ports of the modules
21 declare -A ports=( ["testapi"]="8082" ["reporting"]="8084")
23 ## Urls to check if the modules are deployed or not ?
24 #declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \
25 # ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html")
27 declare -A urls=( ["testapi"]="http://localhost:8082/" \
28 ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html")
31 ### Functions related to checking.
33 function is_deploying() {
34 echo -e "Checking job statuses"
35 for module in "${modules[@]}"
37 if get_status $module; then
43 function get_status() {
44 xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1")
45 building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml")
46 if [[ $building == "false" ]]
54 function get_docker_status() {
55 status=$(service docker status | sed -n 3p | cut -d ' ' -f5)
56 echo -e "Docker status: $status"
57 if [ $status = "active" ]
65 function check_connectivity() {
66 echo "Checking $1 connection : $2"
67 cmd=`curl --head -m10 --request GET ${2} | grep '200 OK' > /dev/null`
69 if [[ $rc == 0 ]]; then
76 function check_modules() {
77 echo -e "Checking modules"
79 for module in "${modules[@]}"
81 if ! check_connectivity $module "${urls[$module]}"; then
82 echo -e "$module failed"
83 failed_modules+=($module)
86 if [ ! -z "$failed_modules" ]; then
87 echo -e "Failed Modules: $failed_modules"
90 echo -e "All modules working good"
95 ### Functions related fixes.
97 function restart_docker_fix() {
98 echo -e "Running restart_docker_fix"
99 service docker restart
100 start_containers_fix "${modules[@]}"
103 function docker_proxy_fix() {
104 echo -e "Running docker_proxy_fix"
106 for module in "${fix_modules[@]}"
108 echo -e "Kill docker proxy and restart containers"
109 pid=$(netstat -nlp | grep :${ports[$module]} | awk '{print $7}' | cut -d'/' -f1)
111 if [ ! -z "$pid" ]; then
113 start_containers_fix $module
118 function start_containers_fix() {
119 echo "Runnning start_containers_fix"
120 start_modules=("${@}")
121 for module in "${start_modules[@]}"
123 echo -e "Starting a container $module"
124 sudo docker stop $module
125 sudo docker start $module
127 if ! check_connectivity $module "${urls[$module]}"; then
128 echo -e "Starting an old container $module_old"
129 sudo docker stop $module
130 sudo docker start $module"_old"
139 echo -e "WatchDog Started"
141 echo -e `date "+%Y-%m-%d %H:%M:%S.%N"`
144 if ! is_deploying; then
145 echo -e "Jenkins Jobs running"
149 ## If the problem is related to docker daemon
151 if get_docker_status; then
153 if ! check_modules; then
154 echo -e "Watchdog failed while restart_docker_fix"
159 ## If the problem is related to docker containers
161 if ! check_modules; then
162 start_containers_fix "${failed_modules[@]}"
165 ## If the problem is related to docker proxy
167 if ! check_modules; then
168 docker_proxy_fix "${failed_modules[@]}"
171 ## If nothing works out
173 if ! check_modules; then
174 echo -e "Watchdog failed"