Code Review / releng.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | patch | inline | side by side (parent: 8c47f58)
jjb: infra: bifrost-upload-logs.sh: Consolidate upload step to main job 75/27175/3
authorMarkos Chandras <mchandras@suse.de>
Wed, 18 Jan 2017 14:54:07 +0000 (14:54 +0000)
committerMarkos Chandras <mchandras@suse.de>
Thu, 19 Jan 2017 23:51:28 +0000 (23:51 +0000)
In case of job failures, the upload builder is not executed so the
published links are dead. We move the upload step to the EXIT handler
which is always executed despite the main job exit code.

Moreover, The gsutil tool can compress the files itself but in such
a way that the web browser can still display the contents without
the need for the user to download and uncompress the file himself.
This makes debugging more convenient. We also make the whole upload
process somewhat less noisy. Finally, we include the
$GERRIT_PATCHSET_NUMBER in the title and hyperlink since we provide
and retain build results for each patchset instead of a single set
just for the latest one.

Change-Id: I43efd519a5743661d1f258aa338fb0769f59b6ed
Suggested-by: Julien Zhang <zhang.jun3g@zte.com.cn> # compression only
Signed-off-by: Markos Chandras <mchandras@suse.de>
jjb/infra/bifrost-upload-logs.sh [deleted file] patch | blob | history
jjb/infra/bifrost-verify-jobs.yml patch | blob | history
jjb/infra/bifrost-verify.sh patch | blob | history

diff --git a/jjb/infra/bifrost-upload-logs.sh b/jjb/infra/bifrost-upload-logs.sh
deleted file mode 100755 (executable)
index 8e1982d..0000000
--- a/jjb/infra/bifrost-upload-logs.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-# SPDX-license-identifier: Apache-2.0
-##############################################################################
-# Copyright (c) 2016 SUSE.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-set -eu
-set -o pipefail
-
-BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText"
-BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:}
-BIFROST_COMPRESS_SUFFIX="tar.gz"
-BIFROST_COMPRESSED_LOGS=()
-
-echo "Uploading build logs to ${BIFROST_LOG_URL}"
-
-echo "Uploading console output"
-curl -L ${BIFROST_CONSOLE_LOG} | gsutil cp - ${BIFROST_GS_URL}/build_log.txt
-
-[[ ! -d ${WORKSPACE}/logs ]] && exit 0
-
-pushd ${WORKSPACE}/logs/ &> /dev/null
-for x in *.log; do
-    echo "Compressing and uploading $x"
-    tar -czf - $x | gsutil cp - ${BIFROST_GS_URL}/${x}.${BIFROST_COMPRESS_SUFFIX} 1>/dev/null
-    BIFROST_COMPRESSED_LOGS+=(${x}.${BIFROST_COMPRESS_SUFFIX})
-done
-popd &> /dev/null
-
-echo "Generating the landing page"
-cat > index.html << EOF
-<html>
-<h1>Build results for <a href=https://$GERRIT_NAME/#/c/$GERRIT_CHANGE_NUMBER>$GERRIT_NAME/$GERRIT_CHANGE_NUMBER</a></h1>
-<h2>Job: $JOB_NAME</h2>
-<ul>
-<li><a href=${BIFROST_LOG_URL}/build_log.txt>build_log.txt</a></li>
-EOF
-
-for x in ${BIFROST_COMPRESSED_LOGS[@]}; do
-    echo "<li><a href=${BIFROST_LOG_URL}/${x}>${x}</a></li>" >> index.html
-done
-
-cat >> index.html << EOF
-</ul>
-</html>
-EOF
-
-gsutil cp index.html ${BIFROST_GS_URL}/index.html
diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml
index 6232859..8184b63 100644 (file)
--- a/jjb/infra/bifrost-verify-jobs.yml
+++ b/jjb/infra/bifrost-verify-jobs.yml
@@ -142,7 +142,6 @@
     builders:
         - bifrost-set-name
         - bifrost-build
-        - bifrost-artifacts-upload
 
     publishers:
         - email:
@@ -220,9 +219,3 @@
     builders:
         - shell:
             !include-raw: ./bifrost-verify.sh
-
-- builder:
-    name: bifrost-artifacts-upload
-    builders:
-        - shell:
-            !include-raw: ./bifrost-upload-logs.sh
diff --git a/jjb/infra/bifrost-verify.sh b/jjb/infra/bifrost-verify.sh
index 9fbb1d0..2cb41a4 100755 (executable)
--- a/jjb/infra/bifrost-verify.sh
+++ b/jjb/infra/bifrost-verify.sh
@@ -11,7 +11,49 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-trap fix_ownership EXIT
+trap cleanup_and_upload EXIT
+
+function upload_logs() {
+    BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText"
+    BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:}
+
+    echo "Uploading build logs to ${BIFROST_LOG_URL}"
+
+    echo "Uploading console output"
+    curl -s -L ${BIFROST_CONSOLE_LOG} | gsutil -q cp -Z - ${BIFROST_GS_URL}/build_log.txt
+
+    [[ ! -d ${WORKSPACE}/logs ]] && exit 0
+
+    pushd ${WORKSPACE}/logs/ &> /dev/null
+    for x in *.log; do
+        echo "Compressing and uploading $x"
+        gsutil -q cp -Z ${x} ${BIFROST_GS_URL}/${x}
+    done
+
+    echo "Generating the landing page"
+    cat > index.html <<EOF
+<html>
+<h1>Build results for <a href=https://$GERRIT_NAME/#/c/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER>$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER</a></h1>
+<h2>Job: $JOB_NAME</h2>
+<ul>
+<li><a href=${BIFROST_LOG_URL}/build_log.txt>build_log.txt</a></li>
+EOF
+
+    for x in *.log; do
+        echo "<li><a href=${BIFROST_LOG_URL}/${x}>${x}</a></li>" >> index.html
+    done
+
+    cat >> index.html << EOF
+</ul>
+</html>
+EOF
+
+    gsutil -q cp index.html ${BIFROST_GS_URL}/index.html
+
+    rm index.html
+
+    popd &> /dev/null
+}
 
 function fix_ownership() {
     if [ -z "${JOB_URL+x}" ]; then
@@ -25,6 +67,13 @@ function fix_ownership() {
     fi
 }
 
+function cleanup_and_upload() {
+    original_exit=$?
+    fix_ownership
+    upload_logs
+    exit $original_exit
+}
+
 # check distro to see if we support it
 if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then
     echo "Distro $DISTRO is not supported!"