bugfix: autoupdate failed due to mingle with watchdog

1. In the original design, if the auto-job starts soon after
the watchdog starts, the auto update might be mingled with
connectivity check, which may lead to update failure. so instead,
the connectivity is checked first, if failed and module is not in deploying,
restart module.
2. only automate-docker-deploy job will impact the container status during
auto-job, so substitue auto-job status check with automate-docker-deploy's
3. the watchdog is not only for testapi, but also for reporting, and
all docker container server deployed in testresults.opnfv.org, so move it under
utils/test/opts, which is leveraged to store tool-sets employed for testing tools.

Change-Id: I766f3a534a3d510ce7509d4e742150150ccd8f54
Signed-off-by: SerenaFeng <feng.xiaowei@zte.com.cn>

diff --git a/utils/test/testapi/tools/watchdog/docker_watch.sh b/utils/test/opts/watchdog.sh
similarity index 92%
rename from utils/test/testapi/tools/watchdog/docker_watch.sh
rename to utils/test/opts/watchdog.sh
index f1d8946..51868d7 100644 (file)
--- a/utils/test/testapi/tools/watchdog/docker_watch.sh
+++ b/utils/test/opts/watchdog.sh
@@ -27,7 +27,7 @@ declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \
 ### Functions related to checking.
 
 function is_deploying() {
-    xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1")
+    xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-docker-deploy-master/lastBuild/api/xml?depth=1")
     building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml")
     if [[ $building == "false" ]]
     then
@@ -64,12 +64,11 @@ function check_modules() {
     failed_modules=()
     for module in "${modules[@]}"
     do
-        if is_deploying $module; then
-            continue
-        fi
         if ! check_connectivity $module "${urls[$module]}"; then
-            echo -e "$module failed"
-            failed_modules+=($module)
+            if ! is_deploying $module; then
+                echo -e "$module failed"
+                failed_modules+=($module)
+            fi
         fi
     done
     if [ ! -z "$failed_modules" ]; then
@@ -114,13 +113,11 @@ function start_containers_fix() {
 
 function start_container_fix() {
     echo -e "Starting a container $module"
-    sudo docker stop $module
-    sudo docker start $module
+    sudo docker restart $module
     sleep 5
     if ! check_connectivity $module "${urls[$module]}"; then
         echo -e "Starting an old container $module_old"
-        sudo docker stop $module
-        sudo docker start $module"_old"
+        sudo docker restart $module"_old"
         sleep 5
     fi
 }