Merge "Bottlenecks: run env preparation inside the container"

diff --git a/jjb/armband/armband-project-jobs.yml b/jjb/armband/armband-project-jobs.yml
index f91e9b1..b40949d 100644 (file)
--- a/jjb/armband/armband-project-jobs.yml
+++ b/jjb/armband/armband-project-jobs.yml
@@ -97,6 +97,8 @@
     builders:
         - shell:
             !include-raw-escape: ./build.sh
+        - shell:
+            !include-raw-escape: ./upload-artifacts.sh
 
     publishers:
         - email:

diff --git a/jjb/armband/upload-artifacts.sh b/jjb/armband/upload-artifacts.sh
new file mode 100755 (executable)
index 0000000..f4e84e9
--- /dev/null
+++ b/jjb/armband/upload-artifacts.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o pipefail
+
+# check if we built something
+if [ -f $WORKSPACE/.noupload ]; then
+    echo "Nothing new to upload. Exiting."
+    /bin/rm -f $WORKSPACE/.noupload
+    exit 0
+fi
+
+# source the opnfv.properties to get ARTIFACT_VERSION
+source $WORKSPACE/opnfv.properties
+
+# storing ISOs for verify & merge jobs will be done once we get the disk array
+if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then
+    # store ISO locally on NFS first
+    ISOSTORE="/home/jenkins/opnfv/iso_store"
+    if [[ -d "$ISOSTORE" ]]; then
+        # remove all but most recent 3 ISOs first to keep iso_mount clean & tidy
+        cd $ISOSTORE
+        ls -tp | grep -v '/' | tail -n +4 | xargs -d '\n' /bin/rm -f --
+
+        # store ISO
+        echo "Storing latest ISO in local storage"
+        touch .storing
+        /bin/cp -f $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso \
+            $ISOSTORE/opnfv-$OPNFV_ARTIFACT_VERSION.iso
+        rm .storing
+    fi
+fi
+
+# log info to console
+echo "Uploading armband artifacts. This could take some time..."
+echo
+
+echo "Started at $(date)"
+cd $WORKSPACE
+# upload artifact and additional files to google storage
+gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
+gsutil cp $WORKSPACE/opnfv.properties \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
+if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then
+    gsutil cp $WORKSPACE/opnfv.properties \
+    gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
+elif [[ "$JOB_NAME" =~ "merge" ]]; then
+    echo "Uploaded Armband Fuel ISO for a merged change"
+fi
+echo "Ended at $(date)"
+
+gsutil -m setmeta \
+    -h "Content-Type:text/html" \
+    -h "Cache-Control:private, max-age=0, no-transform" \
+    gs://$GS_URL/latest.properties \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1
+
+gsutil -m setmeta \
+    -h "Cache-Control:private, max-age=0, no-transform" \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1
+
+# disabled errexit due to gsutil setmeta complaints
+#   BadRequestException: 400 Invalid argument
+# check if we uploaded the file successfully to see if things are fine
+gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1
+if [[ $? -ne 0 ]]; then
+    echo "Problem while uploading artifact!"
+    echo "Check log $WORKSPACE/gsutil.iso.log on the machine where this build is done."
+    exit 1
+fi
+
+echo "Done!"
+echo
+echo "--------------------------------------------------------"
+echo
+echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
+echo
+echo "--------------------------------------------------------"
+echo

diff --git a/jjb/opnfv/opnfv-docker.sh b/jjb/opnfv/opnfv-docker.sh
index 8f44268..702c0ce 100644 (file)
--- a/jjb/opnfv/opnfv-docker.sh
+++ b/jjb/opnfv/opnfv-docker.sh
@@ -17,6 +17,14 @@ echo "--------------------------------------------------------"
 echo
 
 
+if [[ -n $(ps -ef|grep 'docker build'|grep -v grep) ]]; then
+    echo "There is already another build process in progress:"
+    echo $(ps -ef|grep 'docker build'|grep -v grep)
+    # Abort this job since it will colide and might mess up the current one.
+    echo "Aborting..."
+    exit 1
+fi
+
 # Remove previous running containers if exist
 if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then
     echo "Removing existing $DOCKER_REPO_NAME containers..."

diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml
index 5701ab9..7fb4d65 100644 (file)
--- a/jjb/releng-macros.yaml
+++ b/jjb/releng-macros.yaml
@@ -339,17 +339,27 @@
             # generate and upload lint log
             echo "Running flake8 code on $PROJECT ..."
 
+            # Get number of flake8 violations. If none, this will be an
+            # empty string: ""
             FLAKE_COUNT="$(find . \
                 -path './releng_flake8' -prune -o \
                 -type f -name "*.py" -print | \
                 xargs flake8 --exit-zero -qq --count 2>&1)"
-            echo -e "Flake8 Violations: $FLAKE_COUNT\n---" >> lint.log
-            # At some point this need to handle the case of finding no
-            # violations.
-            find . \
-                -path './releng_flake8' -prune -o \
-                -type f -name "*.py" -print | \
-                xargs flake8 --exit-zero --first >> lint.log
+
+            if [ ! -z $FLAKE_COUNT ]; then
+              echo "Flake8 Violations: $FLAKE_COUNT" >> lint.log
+              find . \
+                  -path './releng_flake8' -prune -o \
+                  -type f -name "*.py" -print | \
+                  xargs flake8 --exit-zero --first >> violation.log
+              SHOWN=$(wc -l violation.log | cut -d' ' -f1)
+              echo -e "First $SHOWN shown\n---" >> lint.log
+              cat violation.log >> lint.log
+              sed -r -i '4,$s/^/ /g' lint.log
+              rm violation.log
+            else
+              echo -e "Flake8 Violations: 0" > lint.log
+            fi
 
             deactivate
 
@@ -363,7 +373,6 @@
             set -o xtrace
             export PATH=$PATH:/usr/local/bin/
             if [[ -e lint.log ]] ; then
-                sed -r -i '3,$s/^/ /g' lint.log
                 echo -e "\nposting linting report to gerrit...\n"
                 cat lint.log
                 echo