From 1cee98670e33ef88e6cbdb69e19ca574462b019e Mon Sep 17 00:00:00 2001 From: SerenaFeng Date: Thu, 21 Sep 2017 11:16:44 +0800 Subject: [PATCH] bugfix: autoupdate failed due to mingle with watchdog 1. In the original design, if the auto-job starts soon after the watchdog starts, the auto update might be mingled with connectivity check, which may lead to update failure. so instead, the connectivity is checked first, if failed and module is not in deploying, restart module. 2. only automate-docker-deploy job will impact the container status during auto-job, so substitue auto-job status check with automate-docker-deploy's 3. the watchdog is not only for testapi, but also for reporting, and all docker container server deployed in testresults.opnfv.org, so move it under utils/test/opts, which is leveraged to store tool-sets employed for testing tools. Change-Id: I766f3a534a3d510ce7509d4e742150150ccd8f54 Signed-off-by: SerenaFeng --- .../tools/watchdog/docker_watch.sh => opts/watchdog.sh} | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) rename utils/test/{testapi/tools/watchdog/docker_watch.sh => opts/watchdog.sh} (92%) diff --git a/utils/test/testapi/tools/watchdog/docker_watch.sh b/utils/test/opts/watchdog.sh similarity index 92% rename from utils/test/testapi/tools/watchdog/docker_watch.sh rename to utils/test/opts/watchdog.sh index f1d8946b6..51868d709 100644 --- a/utils/test/testapi/tools/watchdog/docker_watch.sh +++ b/utils/test/opts/watchdog.sh @@ -27,7 +27,7 @@ declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ ### Functions related to checking. function is_deploying() { - xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1") + xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-docker-deploy-master/lastBuild/api/xml?depth=1") building=$(grep -oPm1 "(?<=)[^<]+" <<< "$xml") if [[ $building == "false" ]] then @@ -64,12 +64,11 @@ function check_modules() { failed_modules=() for module in "${modules[@]}" do - if is_deploying $module; then - continue - fi if ! check_connectivity $module "${urls[$module]}"; then - echo -e "$module failed" - failed_modules+=($module) + if ! is_deploying $module; then + echo -e "$module failed" + failed_modules+=($module) + fi fi done if [ ! -z "$failed_modules" ]; then @@ -114,13 +113,11 @@ function start_containers_fix() { function start_container_fix() { echo -e "Starting a container $module" - sudo docker stop $module - sudo docker start $module + sudo docker restart $module sleep 5 if ! check_connectivity $module "${urls[$module]}"; then echo -e "Starting an old container $module_old" - sudo docker stop $module - sudo docker start $module"_old" + sudo docker restart $module"_old" sleep 5 fi } -- 2.16.6