X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=fuel%2Fci%2Fbuild.sh;h=f8e164a76ed6c3affb915ad87cf4d042cf2ca0c4;hb=563547b4a9f44090f32c0e17d040114854563760;hp=bb0d124432fdd837d0609639f52c6f252d2c3fab;hpb=df6c1db2695030b1ba68184bac9aab9fb2d41e71;p=genesis.git diff --git a/fuel/ci/build.sh b/fuel/ci/build.sh index bb0d124..f8e164a 100755 --- a/fuel/ci/build.sh +++ b/fuel/ci/build.sh @@ -17,6 +17,9 @@ if [ -f ${LOCK_FILE} ]; then \ fi; \ fi;' EXIT +############################################################################ +# BEGIN of usage description +# usage () { cat << EOF @@ -31,6 +34,8 @@ OPTIONS: -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal -v version tag to be applied to the build result -r alternative remote access method script/program. curl is default. + -t run small build-script unit test. + -T run large build-script unit test. -f build flags ($BUILD_FLAGS): o s: Do nothing, succeed o f: Do nothing, fail @@ -70,156 +75,67 @@ build -c http://opnfv.org/artifactory/fuel/cache -d ~/jenkins/genesis/fuel/ci/ou NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work EOF } - - - -debug_make () { - make -C ${BUILD_BASE} clean - echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso - echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions - rm -rf ${BUILD_BASE}/release - mkdir ${BUILD_BASE}/release - echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso - echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt - echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history - echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log -} - -cache_check () { - if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \ - [ ! -f ${BUILD_BASE}/.versions ] || \ - [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \ - [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then - echo "Cache not present in the build system" - echo "TEST FAILED" - exit $rc - fi -} - -integration-test () { - -##### Always succeed integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - ALWAYS SUCCEED" - set +e - $0 -f s tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=150 - exit $rc - fi - -##### Always fail integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - ALWAYS FAIL" - set +e - $0 -f f tmp/output - rc=$? - set -e - if [ $rc -eq 0 ]; then - echo "TEST FAILED" - rc=151 - exit $rc - fi - -##### Populate cache integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - POPULATE CACHE" - rm -rf tmp - mkdir tmp - mkdir tmp/cache - set +e - $0 -c ${BUILD_CACHE_URI} -f PD tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=152 - exit $rc - fi - - if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then - echo "TEST FAILED" - rc=153 - exit $rc - fi - rc=154 - cache_check - -##### Build uinge cache integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - BUILD USING CACHE" - set +e - $0 -c ${BUILD_CACHE_URI} -f D tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=155 - exit $rc - fi - - if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then - echo "TEST FAILED" - rc=156 - exit $rc - fi - rc=157 - cache_check - -#### Repopulate cache because cache non existant #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - BUILD USING CACHE" - rm -rf tmp/cache/* - set +e - $0 -c ${BUILD_CACHE_URI} -f D tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=158 - exit $rc - fi - - if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then - echo "TEST FAILED" - rc=160 - exit $rc - fi - rc=161 - cache_check - -# Repopulate cache because cach is ivalidated -#TBD -rm -rf tmp -echo "All tests passed!" -rc=0 -exit $rc +# +# END of usage description +############################################################################ + +############################################################################ +# Begin of string xor function +# +function xor() +{ + local res=(`echo "$1" | sed "s/../0x& /g"`) + shift 1 + while [[ "$1" ]]; do + local one=(`echo "$1" | sed "s/../0x& /g"`) + local count1=${#res[@]} + if [ $count1 -lt ${#one[@]} ] + then + count1=${#one[@]} + fi + for (( i = 0; i < $count1; i++ )) + do + res[$i]=$((${one[$i]:-0} ^ ${res[$i]:-0})) + done + shift 1 + done + printf "%02x" "${res[@]}" } +# +# END of string xor function +############################################################################ -#DEFAULT VALUES +############################################################################ +# BEGIN of variables to customize +# BUILD_BASE=$(readlink -e ../build/) RESULT_DIR="${BUILD_BASE}/release" BUILD_SPEC="${BUILD_BASE}/config.mk" CACHE_DIR="cache" LOCAL_CACHE_ARCH_NAME="fuel-cache" -REMOTE_CACHE_ARCH_NAME="fuel_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")" -REMOTE_ACCESS_METHD=curl -#SCRIPT ASSIGNED VARIABLES +REMOTE_ACCESS_METHD=curl +INCLUDE_DIR=../include +# +# END of variables to customize +############################################################################ + +############################################################################ +# BEGIN of script assigned variables +# SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) LOCK_FILE="${SCRIPT_DIR}/.build.lck" +CACHE_TMP="${SCRIPT_DIR}/tmp" TEST_SUCCEED=0 TEST_FAIL=0 UNIT_TEST=0 UPDATE_CACHE=0 POPULATE_CACHE=0 -RECURSIV=0 +RECURSIVE=0 DETACH=0 DEBUG=0 INTEGRATION_TEST=0 +FULL_INTEGRATION_TEST=0 INTERACTIVE=0 BUILD_CACHE_URI= BUILD_SPEC= @@ -227,52 +143,79 @@ BUILD_DIR= BUILD_LOG= BUILD_VERSION= MAKE_ARGS= - -while getopts "s:c:v:f:l:r:RTh" OPTION +FUEL_GIT_SRC="$(make -f ../build/config.mk get-fuel-repo | cut -d " " -f1)" +FUEL_GIT_BRANCH="$(make -f ../build/config.mk get-fuel-repo | cut -d " " -f2)" +CACHE_MD5=$(md5sum ../build/cache.mk | cut -f1 -d " ") +CONFIG_MD5=$(md5sum ../build/config.mk | cut -f1 -d " ") +FUEL_COMMIT_ID=$(git ls-remote $FUEL_GIT_SRC -t $FUEL_GIT_BRANCH | cut -d $'\t' -f1) +REMOTE_CACHE_ARCH_HASH_TMP="$(xor $CACHE_MD5 $CONFIG_MD5)" +REMOTE_CACHE_ARCH_HASH="$(xor $REMOTE_CACHE_ARCH_HASH_TMP $FUEL_COMMIT_ID)" +REMOTE_CACHE_ARCH_NAME="fuel_cache-$REMOTE_CACHE_ARCH_HASH" +# +# END of script assigned variables +############################################################################ + +############################################################################ +# BEGIN of include pragmas +# +source ${INCLUDE_DIR}/build.sh.debug +# +# END of include +############################################################################ + +############################################################################ +# BEGIN of main +# +while getopts "s:c:v:f:l:r:RtTh" OPTION do case $OPTION in - h) - usage - rc=0 - exit $rc - ;; - - s) - BUILD_SPEC=${OPTARG} - ;; - - c) - BUILD_CACHE_URI=${OPTARG} - ;; - - l) - BUILD_LOG=${OPTARG} - ;; - - v) - BUILD_VERSION=${OPTARG} - ;; - - f) - BUILD_FLAGS=${OPTARG} - ;; - - r) REMOTE_ACCESS_METHD=${OPTARG} - ;; - - R) - RECURSIVE=1 - ;; - - T) - INTEGRATION_TEST=1 - ;; - - *) - echo "${OPTION} is not a valid argument" - rc=100 - exit $rc - ;; + h) + usage + rc=0 + exit $rc + ;; + + s) + BUILD_SPEC=${OPTARG} + ;; + + c) + BUILD_CACHE_URI=${OPTARG} + ;; + + l) + BUILD_LOG=${OPTARG} + ;; + + v) + BUILD_VERSION=${OPTARG} + ;; + + f) + BUILD_FLAGS=${OPTARG} + ;; + + r) REMOTE_ACCESS_METHD=${OPTARG} + ;; + + R) + RECURSIVE=1 + ;; + + t) + INTEGRATION_TEST=1 + ;; + + T) + INTEGRATION_TEST=1 + FULL_INTEGRATION_TEST=1 + ;; + + *) + echo "${OPTION} is not a valid argument" + rc=100 + exit $rc + ;; esac done @@ -282,44 +225,44 @@ fi for ((i=0; i<${#BUILD_FLAGS};i++)); do case ${BUILD_FLAGS:$i:1} in - s) - rc=0 - exit $rc - ;; - - f) - rc=1 - exit $rc - ;; - - t) - UNIT_TEST=1 - ;; - - i) - INTERACTIVE=1 - ;; - - P) - POPULATE_CACHE=1 - ;; - - d) - DETACH=1 - echo "Detach is not yet supported - exiting ...." - rc=100 - exit $rc - ;; - - D) - DEBUG=1 - ;; - - *) - echo "${BUILD_FLAGS:$i:1} is not a valid build flag" - rc=100 - exit $rc - ;; + s) + rc=0 + exit $rc + ;; + + f) + rc=1 + exit $rc + ;; + + t) + UNIT_TEST=1 + ;; + + i) + INTERACTIVE=1 + ;; + + P) + POPULATE_CACHE=1 + ;; + + d) + DETACH=1 + echo "Detach is not yet supported - exiting ...." + rc=100 + exit $rc + ;; + + D) + DEBUG=1 + ;; + + *) + echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...." + rc=100 + exit $rc + ;; esac done @@ -330,26 +273,26 @@ if [ ${INTEGRATION_TEST} -eq 1 ]; then fi if [ ! -f ${BUILD_SPEC} ]; then - echo "spec file does not exist: $BUILD_SPEC" + echo "spec file does not exist: $BUILD_SPEC - exiting ...." rc=100 exit $rc fi if [ -z ${BUILD_DIR} ]; then - echo "Missing build directory" + echo "Missing build directory - exiting ...." rc=100 exit $rc fi if [ ! -z ${BUILD_LOG} ]; then if [[ ${RECURSIVE} -ne 1 ]]; then - set +e - eval $0 -R $@ > ${BUILD_LOG} 2>&1 - rc=$? - set -e - if [ $rc -ne 0]; then - exit $rc - fi + set +e + eval $0 -R $@ > ${BUILD_LOG} 2>&1 + rc=$? + set -e + if [ $rc -ne 0]; then + exit $rc + fi fi fi @@ -375,46 +318,86 @@ echo $$ > ${LOCK_FILE} if [ ! -z ${BUILD_CACHE_URI} ]; then if [ ${POPULATE_CACHE} -ne 1 ]; then - rm -rf /tmp/cache - mkdir /tmp/cache - echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..." - set +e - ${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Remote cache does not exist, or is not accessible - a new cache will be built ..." - POPULATE_CACHE=1 - else - echo "Unpacking cache file ..." - tar -C /tmp/cache -xvf /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz - cp /tmp/cache/cache/.versions ${BUILD_BASE}/. - echo "Validating cache content ..." - set +e - make -C ${BUILD_BASE} validate-cache; - rc=$? - if [ $rc -ne 0 ]; then - echo "Cache invalid - a new cache will be built " - POPULATE_CACHE=1 - else - cp -rf /tmp/cache/cache/. ${BUILD_BASE} - fi - rm -rf /tmp/cache - fi + rm -rf ${CACHE_TMP}/cache + mkdir -p ${CACHE_TMP}/cache + echo "Downloading cache archive ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..." + set +e + ${REMOTE_ACCESS_METHD} -o ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Remote cache does not exist, or is not accessible - a new cache will be built ..." + POPULATE_CACHE=1 + else + echo "Unpacking cache archive ..." + set +e + tar -C ${CACHE_TMP}/cache -xvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "WARNING: The cache seems to be corrupt or has trailing garbage, will try to use brute force" + echo "Info about the cache below:" + set +e + file ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz + tar -C ${CACHE_TMP}/cache -tvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz + set -e + echo "Current time is: `date`" + set +e + pushd ${CACHE_TMP}/cache + gunzip -dcq ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz | tar -xvf - + rc=$? + set -e + popd + if [ $rc -ne 0 ]; then + echo "ERROR: Not able to resolve the cache corruption" + POPULATE_CACHE=1 + else + echo "The chache corruption was resolved" + cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/. + set +e + make -C ${BUILD_BASE} validate-cache; + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Cache invalid - a new cache will be built " + POPULATE_CACHE=1 + else + echo "Cache is up to date and will be used" + cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE} + fi + fi + else + echo "Cache archive is intact" + cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/. + set +e + make -C ${BUILD_BASE} validate-cache; + rc=$? + set -e + + if [ $rc -ne 0 ]; then + echo "Cache invalid - a new cache will be built " + POPULATE_CACHE=1 + else + echo "Cache is up to date and will be used" + cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE} + fi + fi + rm -rf ${CACHE_TMP}/cache + fi fi fi if [ ${POPULATE_CACHE} -eq 1 ]; then if [ ${DEBUG} -eq 0 ]; then - set +e - cd ${BUILD_BASE} && make clean - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build - make clean failed, exiting ..." - rc=100 - exit $rc - fi + set +e + cd ${BUILD_BASE} && make clean + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build - make clean failed, exiting ..." + rc=100 + exit $rc + fi fi fi @@ -442,12 +425,12 @@ if [ ${DEBUG} -eq 0 ]; then rc=$? set -e if [ $rc -gt 0 ]; then - echo "Build: make all failed, exiting ..." - rc=200 - exit $rc + echo "Build: make all failed, exiting ..." + rc=200 + exit $rc fi else -debug_make + debug_make fi set +e make -C ${BUILD_BASE} prepare-cache @@ -455,24 +438,36 @@ rc=$? set -e if [ $rc -gt 0 ]; then - echo "Build: make prepare-cache failed, exiting ..." + echo "Build: make prepare-cache failed - exiting ..." rc=100 exit $rc fi echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..." rm -rf ${BUILD_DIR} -mkdir ${BUILD_DIR} +mkdir -p ${BUILD_DIR} cp ${BUILD_BASE}/.versions ${BUILD_DIR} cp ${RESULT_DIR}/*.iso* ${BUILD_DIR} if [ $POPULATE_CACHE -eq 1 ]; then if [ ! -z ${BUILD_CACHE_URI} ]; then - echo "Building cache ..." - tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} - echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}" - ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} - rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz + echo "Building cache ..." + tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} + set +e + tar -C ${CACHE_TMP}/cache -tvf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "WARNING the cache archive generated seems to be corrupt, or containing trailing garbage" + else + echo "The Cache archive build is intact" + fi + echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}" + ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz + rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz fi fi echo "Success!!!" exit 0 +# +# END of main +############################################################################