From fe936cb574e7b72a59c4f32b6096a32763e195c8 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Fri, 24 Mar 2017 22:17:15 +0000 Subject: [PATCH] jjb: xci: bifrost-cleanup: Retry 'gsutil rm' on failures Sometimes 'gsutil rm' may fail for unknown reasons like the following one: [...] / [7/10 objects] 70% Done / [8/10 objects] 80% Done / [9/10 objects] 90% Done Operation completed over 10 objects. CommandException: 1 files/objects could not be removed. Try a few more times to remove the objects before giving up. Change-Id: I6905a806446dac41ddc43d4195514a5233480ccd Signed-off-by: Markos Chandras --- jjb/xci/bifrost-cleanup-job.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jjb/xci/bifrost-cleanup-job.yml b/jjb/xci/bifrost-cleanup-job.yml index 4e9e2a8d4..d4b2157da 100644 --- a/jjb/xci/bifrost-cleanup-job.yml +++ b/jjb/xci/bifrost-cleanup-job.yml @@ -65,9 +65,16 @@ echo "gsutil will not be executed until this is fixed!" exit 1 fi - # No force (-f). We always verify upstream jobs so if there are no logs - # something else went wrong and we need to break immediately and investigate - gsutil -m rm -r $BIFROST_GS_URL + try_to_rm=1 + while [[ $try_to_rm -lt 6 ]]; do + gsutil -m rm -r $BIFROST_GS_URL && _exitcode=$? && break + _exitcode=$? + echo "gsutil rm failed! Trying again... (attempt #$i)" + let try_to_rm += 1 + # Give it some time... + sleep 10 + done + exit $_exitcode triggers: - '{project}-gerrit-trigger-cleanup': -- 2.16.6