From 2de197be04b871b437e35b55ca7f48d921595303 Mon Sep 17 00:00:00 2001 From: Jonas Bjurel Date: Sun, 22 Mar 2015 23:04:43 +0100 Subject: [PATCH] Fuel build cache handling - Cache invalidation supported in case cache is outdated compared to upstream. - Fully automated build.sh integration tests added (build.sh -t or build.sh -T) - Rebased to Daniel Smiths's patch-set: genesis refs/changes/71/171/4 - Yet untested, will perform automated integration tests over night! - Updated based on Stefan's comments (no more revs.) - Fixed a bug in f_odl_docker/Makefile - validate-cache target added in f_odl_docker/Makefile JIRA: BGS-1 Change-Id: I9f8f30c2a85fd7bf04dab1b566065e5739d4fcb1 Signed-off-by: Jonas Bjurel --- fuel/build/Makefile | 46 ++-- fuel/build/cache.mk | 79 ++++--- fuel/build/config.mk | 9 + fuel/build/f_example_control_bond/Makefile | 4 + fuel/build/f_example_packadd/Makefile | 4 + fuel/build/f_isoroot/Makefile | 4 + fuel/build/f_l23network/Makefile | 4 + fuel/build/f_ntp/Makefile | 4 + fuel/build/f_odl_docker/Makefile | 10 +- fuel/build/f_opnfv_puppet/Makefile | 4 + fuel/build/f_osnaily/Makefile | 4 + fuel/build/f_resolvconf/Makefile | 4 + fuel/build/opendaylight/ | 1 + fuel/build/opendaylight/Makefile | 27 ++- fuel/build/opendaylight/f_odl/Makefile | 4 + fuel/ci/build.sh | 194 +++++----------- fuel/include/build.sh.debug | 355 +++++++++++++++++++++++++++++ 17 files changed, 551 insertions(+), 206 deletions(-) create mode 120000 fuel/build/opendaylight/ create mode 100644 fuel/include/build.sh.debug diff --git a/fuel/build/Makefile b/fuel/build/Makefile index b6b0eac..364fe31 100644 --- a/fuel/build/Makefile +++ b/fuel/build/Makefile @@ -8,11 +8,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +SHELL = /bin/bash ############################################################################ # BEGIN of variables to customize # -SHELL = /bin/bash - #Input args export UNIT_TEST = FALSE export INTERACTIVE = TRUE @@ -31,20 +30,10 @@ export DEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/main export PUPPET_DEST := $(BUILD_BASE)/release/puppet/modules export CACHE_DIR := $(BUILD_BASE)/cache export VERSION_FILE := $(BUILD_BASE)/.versions -DOCKERIMG = opnfv.org/ubuntu-builder:14.04 +export DOCKERIMG = opnfv.org/ubuntu-builder:14.04 export TOPDIR := $(shell pwd) - -# -# END of variables to customize -############################################################################# - -#Include definitions -include config.mk -include cache.mk - - -ORIGDIR := $(TOPDIR)/origiso +#Build subclasses SUBDIRS := f_isoroot SUBDIRS += f_opnfv_puppet @@ -53,32 +42,43 @@ SUBDIRS += f_l23network SUBDIRS += f_resolvconf SUBDIRS += f_ntp SUBDIRS += f_odl_docker - - +#SUBDIRS += f_odl # f_example is only an example of how to generate a .deb package and # should not be enabled in official builds. #SUBDIRS += f_example +ORIGDIR := $(TOPDIR)/origiso +# +# END of variables to customize +############################################################################# + SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) + .PHONY: all all: @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) - @echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(BUILD_BASE)/.versions - @echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(BUILD_BASE)/.versions - @echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(BUILD_BASE)/.versions + @echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE) + @echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE) + @echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE) @make -C docker @docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso - +############################################################################ +# BEGIN of Include definitions +# +include config.mk +include cache.mk +# +# END Include definitions +############################################################################# $(ISOCACHE): # Clone Fuel to non-persistent location and build cd /tmp && git clone $(FUEL_MAIN_REPO) cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG) - @echo "fuel" `git -C /tmp/fuel-main show | grep commit | rev | cut -f1 -d " "` >> ${BUILD_BASE}/.versions - + @echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE) # Setup cgroups for docker-in-docker sudo /root/enable_dockerx2 # Patch to fix race condition when doing "Docker-in-Docker" build @@ -130,7 +130,7 @@ build-clean: $(SUBCLEAN) @rm -f $(NEWISO) .PHONY: clean $(SUBCLEAN) -clean: prepare $(SUBCLEAN) +clean: clean-cache prepare $(SUBCLEAN) $(MAKE) -C patch-packages -f Makefile clean #$(MAKE) -C opendaylight -f Makefile clean @rm -f *.iso diff --git a/fuel/build/cache.mk b/fuel/build/cache.mk index a3f3516..a2aa1d5 100644 --- a/fuel/build/cache.mk +++ b/fuel/build/cache.mk @@ -8,23 +8,22 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +SHELL = /bin/bash +CACHEVALIDATE := $(addsuffix .validate,$(SUBDIRS)) +CACHECLEAN := $(addsuffix .clean,$(CACHEFILES) $(CACHEDIRS)) + ############################################################################ # BEGIN of variables to customize # -SHELL = /bin/bash - -#export BUILD_BASE = $(shell pwd) -#export CACHE_DIR = "$(BUILD_BASE)/cache" - CACHEDIRS := opendaylight/f_odl/package CACHEFILES := opendaylight/.odl-build-history CACHEFILES += opendaylight/.odl-build.log CACHEFILES += .versions -CACHEFILES += fuel-6.0.1.iso -#CACHEFILES += $(ISOSRC) - -CACHECLEAN = $(addsuffix .clean,$(CACHEDIRS)) +CACHEFILES += $(shell basename $(ISOSRC)) +# +# END of variables to customize +############################################################################ .PHONY: prepare-cache prepare-cache: make-cache-dir $(CACHEDIRS) $(CACHEFILES) @@ -34,54 +33,66 @@ make-cache-dir: @rm -rf ${CACHE_DIR} @mkdir ${CACHE_DIR} - .PHONY: clean-cache clean-cache: $(CACHECLEAN) @rm -rf ${CACHE_DIR} .PHONY: $(CACHEDIRS) $(CACHEDIRS): - @mkdir -p $(dir ${CACHE_DIR}/$@) - @if [ ! -d ${BUILD_BASE}/$@ ]; then\ - mkdir -p $(dir ${BUILD_BASE}/$@);\ - ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\ - rm -rf ${BUILD_BASE}/$@;\ - else\ - ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\ + @mkdir -p $(dir $(CACHE_DIR)/$@) + @if [ ! -d $(BUILD_BASE)/$@ ]; then\ + mkdir -p $(BUILD_BASE)/$@;\ fi + @ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@ .PHONY: $(CACHEFILES) $(CACHEFILES): - @mkdir -p $(dir ${CACHE_DIR}/$@) - @if [ ! -f ${BUILD_BASE}/$@ ]; then\ - mkdir $(dir ${BUILD_BASE}/$@);\ - echo " " > ${BUILD_BASE}/$@;\ - ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\ - rm -f ${BUILD_BASE}/$@;\ + @mkdir -p $(dir $(CACHE_DIR)/$@) + @if [ ! -d $(dir $(BUILD_BASE)/$@) ]; then\ + mkdir -p $(dir $(BUILD_BASE)/$@);\ + fi + + @if [ ! -f $(BUILD_BASE)/$@ ]; then\ + echo " " > $(BUILD_BASE)/$@;\ + ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\ + rm -f $(BUILD_BASE)/$@;\ else\ - ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\ + ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\ fi .PHONY: validate-cache -validate-cache: -# if [ $(shell md5sum ${BUILD_BASE}/config.mk) -ne $(shell cat ${CACHE_DIR}/.versions | grep config.mk awk '{print $NF}') ]; then\ +validate-cache: prepare $(CACHEVALIDATE) + @if [[ $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep config.mk | awk '{print $$NF}') ]]; then\ echo "Cache does not match current config.mk definition, cache must be rebuilt";\ exit 1;\ fi; -# if [ $(shell md5sum ${BUILD_BASE}/cache.mk) -ne $(shell cat ${CACHE_DIR}/.versions | grep config.mk awk '{print $NF}') ]; then\ + @if [[ $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep cache.mk | awk '{print $$NF}') ]]; then\ echo "Cache does not match current cache.mk definition, cache must be rebuilt";\ exit 1;\ fi; -# $(MAKE) -C opendaylight validate-cache -# if [ $? -ne 0 ]; then\ - echo "Cache does not match current OpenDaylight version, cach must be rebuilt";\ - exit 1;\ - fi; +# Once the Make structure is refactored, this should go in as a validate-cache +# taget in the fuel Makefile + + @REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \ + if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \ + then \ + REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG) | awk '{print $$(NF-1)}'); \ + fi; \ + if [ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep fuel | awk '{print $$NF}') ]; \ + then \ + echo "Cache does not match upstream Fuel, cache must be rebuilt!"; \ + exit 1; \ + fi + + #$(MAKE) -C opendaylight validate-cache -# $(SUBDIRS) +.PHONY: $(CACHEVALIDATE) +$(CACHEVALIDATE): %.validate: + @echo VALIDATE $(CACHEVALIDATE) + $(MAKE) -C $* -f Makefile validate-cache .PHONY: $(CACHECLEAN) $(CACHECLEAN): %.clean: - rm -f ${CACHE_DIR}/$* + rm -rf ${CACHE_DIR}/$* diff --git a/fuel/build/config.mk b/fuel/build/config.mk index 49b0173..19f502d 100644 --- a/fuel/build/config.mk +++ b/fuel/build/config.mk @@ -16,3 +16,12 @@ FUEL_MAIN_TAG = stable/6.0 DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64 DOCKER_TAG := docker-latest + +.PHONY: get-odl-repo +get-odl-repo: + @echo $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) + +.PHONY: get-fuel-repo +get-fuel-repo: + @echo $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG) + diff --git a/fuel/build/f_example_control_bond/Makefile b/fuel/build/f_example_control_bond/Makefile index c311a2d..0949737 100644 --- a/fuel/build/f_example_control_bond/Makefile +++ b/fuel/build/f_example_control_bond/Makefile @@ -18,6 +18,10 @@ clean: @rm -rf tmp @rm -rf release +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: diff --git a/fuel/build/f_example_packadd/Makefile b/fuel/build/f_example_packadd/Makefile index eadc83f..a64293d 100644 --- a/fuel/build/f_example_packadd/Makefile +++ b/fuel/build/f_example_packadd/Makefile @@ -31,6 +31,10 @@ clean: @rm -rf release @rm -f ../release/packages/ubuntu/pool/main/$(DEB_NAME) +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release:release/pool/main/$(DEB_NAME) diff --git a/fuel/build/f_isoroot/Makefile b/fuel/build/f_isoroot/Makefile index 2fdbbae..bde8e64 100644 --- a/fuel/build/f_isoroot/Makefile +++ b/fuel/build/f_isoroot/Makefile @@ -26,6 +26,10 @@ clean: $(SUBCLEAN) $(SUBCLEAN): %.clean: $(MAKE) -C $* -f Makefile clean +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." + .PHONY: release release:clean $(SUBDIRS) @cp -Rvp release/* ../release/isoroot diff --git a/fuel/build/f_l23network/Makefile b/fuel/build/f_l23network/Makefile index c311a2d..0949737 100644 --- a/fuel/build/f_l23network/Makefile +++ b/fuel/build/f_l23network/Makefile @@ -18,6 +18,10 @@ clean: @rm -rf tmp @rm -rf release +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: diff --git a/fuel/build/f_ntp/Makefile b/fuel/build/f_ntp/Makefile index c311a2d..0949737 100644 --- a/fuel/build/f_ntp/Makefile +++ b/fuel/build/f_ntp/Makefile @@ -18,6 +18,10 @@ clean: @rm -rf tmp @rm -rf release +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: diff --git a/fuel/build/f_odl_docker/Makefile b/fuel/build/f_odl_docker/Makefile index b227bee..6135e71 100755 --- a/fuel/build/f_odl_docker/Makefile +++ b/fuel/build/f_odl_docker/Makefile @@ -12,7 +12,7 @@ TOP := $(shell pwd) BUILDTAG := loving_daniel # Edit this to match the GENESIS / OPNFV in your environment -export OPNFV_PUPPET := /home/lmcdasm/MAR24/genesis/common/puppet-opnfv +export OPNFV_PUPPET := $(BUILD_BASE)/../../common/puppet-opnfv include ../config.mk .PHONY: all @@ -25,7 +25,7 @@ all: @docker save ${BUILDTAG} > puppet/modules/opnfv/odl_docker/odl_docker_image.tar @wget ${DOCKER_REPO}/${DOCKER_TAG} -O puppet/modules/opnfv/odl_docker/docker-latest @echo "OPFNV_PUPPET is: ${OPNFV_PUPPET}" - @cp -Rvp ${OPNFV_PUPPET}/manifests/templates/dockerfile/compute_scripts puppet/modules/opnfv + @cp -Rvp ${OPNFV_PUPPET}/manifests/templates/dockerfile/container_scripts puppet/modules/opnfv .PHONY: clean clean: @@ -39,9 +39,13 @@ build-clean: @rm -rf puppet/modules/opnfv/odl_docker/odl_docker_image.tar @rm -rf puppet/modules/opnfv/odl_docker/docker-latest +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: # Fetch PP from OPNFV Common - @cp -Rvp ${OPNFV_PUPPET}/common/puppet-opnfv/manifests/odl_docker.pp ${PUPPET_DEST} + @cp -Rvp ${OPNFV_PUPPET}/manifests/odl_docker.pp ${PUPPET_DEST} @cp -Rvp puppet/modules/* $(PUPPET_DEST) diff --git a/fuel/build/f_opnfv_puppet/Makefile b/fuel/build/f_opnfv_puppet/Makefile index c311a2d..0949737 100644 --- a/fuel/build/f_opnfv_puppet/Makefile +++ b/fuel/build/f_opnfv_puppet/Makefile @@ -18,6 +18,10 @@ clean: @rm -rf tmp @rm -rf release +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: diff --git a/fuel/build/f_osnaily/Makefile b/fuel/build/f_osnaily/Makefile index c311a2d..0949737 100644 --- a/fuel/build/f_osnaily/Makefile +++ b/fuel/build/f_osnaily/Makefile @@ -18,6 +18,10 @@ clean: @rm -rf tmp @rm -rf release +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: diff --git a/fuel/build/f_resolvconf/Makefile b/fuel/build/f_resolvconf/Makefile index c311a2d..0949737 100644 --- a/fuel/build/f_resolvconf/Makefile +++ b/fuel/build/f_resolvconf/Makefile @@ -18,6 +18,10 @@ clean: @rm -rf tmp @rm -rf release +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release: diff --git a/fuel/build/opendaylight/ b/fuel/build/opendaylight/ new file mode 120000 index 0000000..0519ecb --- /dev/null +++ b/fuel/build/opendaylight/ @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fuel/build/opendaylight/Makefile b/fuel/build/opendaylight/Makefile index c112ff6..bd2eeb5 100644 --- a/fuel/build/opendaylight/Makefile +++ b/fuel/build/opendaylight/Makefile @@ -12,7 +12,7 @@ # BEGIN of variables to customize # SHELL = /bin/bash -include ../config.mk + BUILD_DIR := $(shell pwd) GIT_DIR := /tmp @@ -32,14 +32,31 @@ MAVEN_SPEC = $(BUILD_DIR)/odl_maven/settings.xml .PHONY: all all: odl +############################################################################ +# BEGIN of Include definitions +# +include ../config.mk +# +# END Include definitions +############################################################################# + .PHONY: setup setup: rm -f "$(BUILD_BASE)/f_odl" ln -s "$(shell readlink -e $(BUILD_DIR))/f_odl" "$(shell readlink -e $(BUILD_BASE))/f_odl" -.PHONY: check-cache-validity -check-cache-validity: - @if [ $(git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) | awk '{print $(NF-1)}') -ne $(cat $(CACHE_DIR)/.versions | grep odl | awk '{print $NF}') ]; then @echo "Cache is not up to date, stopping!"; exit 1; fi +.PHONY: validate-cache +validate-cache: + @REMOTE_ID=$(shell git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \ + if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \ + then \ + REMOTE_ID=$(shell git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) | awk '{print $$(NF-1)}'); \ + fi; \ + if [ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep odl | awk '{print $$NF}') ]; \ + then \ + echo "Cache does not match upstream OpenDaylight, cache must be rebuilt!"; \ + exit 1; \ + fi .PHONY: odl odl: @@ -69,7 +86,7 @@ else mvn -gs $(MAVEN_SPEC) clean install;\ fi; - @echo "odl" `git -C /tmp/controller show | grep commit | rev | cut -f1 -d " "` >> $(BUILD_BASE)/.versions + @echo "odl" `git -C /tmp/controller show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE) @./make-odl-deb.sh -N $(ODL_SHORT_NAME)_`cd /tmp/controller; git rev-parse --short HEAD` -n $(ODL_SHORT_NAME) -v "$(ODL_VERSION)" -t "$(ODL_MAIN_TAG)" -m $(MAINTAINER) -d $(DEPEND) -p $(TARGET_BUILD_PATH) @echo $(ODL_MAIN_TAG) > .odl-build.log endif diff --git a/fuel/build/opendaylight/f_odl/Makefile b/fuel/build/opendaylight/f_odl/Makefile index 6983f0c..f7ebd3e 100644 --- a/fuel/build/opendaylight/f_odl/Makefile +++ b/fuel/build/opendaylight/f_odl/Makefile @@ -36,6 +36,10 @@ clean: @rm -rf release @rm -f $(DEB_DEST)/$(DEB_NAME).deb +.PHONY: validate-cache +validate-cache: + @echo "No cache validation schema available for $(shell pwd)" + @echo "Continuing ..." .PHONY: release release:release/pool/main/$(DEB_NAME).deb diff --git a/fuel/ci/build.sh b/fuel/ci/build.sh index bb0d124..cdaa6ef 100755 --- a/fuel/ci/build.sh +++ b/fuel/ci/build.sh @@ -17,6 +17,9 @@ if [ -f ${LOCK_FILE} ]; then \ fi; \ fi;' EXIT +############################################################################ +# BEGIN of usage description +# usage () { cat << EOF @@ -31,6 +34,8 @@ OPTIONS: -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal -v version tag to be applied to the build result -r alternative remote access method script/program. curl is default. + -t run small build-script unit test. + -T run large build-script unit test. -f build flags ($BUILD_FLAGS): o s: Do nothing, succeed o f: Do nothing, fail @@ -70,136 +75,13 @@ build -c http://opnfv.org/artifactory/fuel/cache -d ~/jenkins/genesis/fuel/ci/ou NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work EOF } +# +# END of usage description +############################################################################ - - -debug_make () { - make -C ${BUILD_BASE} clean - echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso - echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions - rm -rf ${BUILD_BASE}/release - mkdir ${BUILD_BASE}/release - echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso - echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt - echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history - echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log -} - -cache_check () { - if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \ - [ ! -f ${BUILD_BASE}/.versions ] || \ - [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \ - [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then - echo "Cache not present in the build system" - echo "TEST FAILED" - exit $rc - fi -} - -integration-test () { - -##### Always succeed integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - ALWAYS SUCCEED" - set +e - $0 -f s tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=150 - exit $rc - fi - -##### Always fail integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - ALWAYS FAIL" - set +e - $0 -f f tmp/output - rc=$? - set -e - if [ $rc -eq 0 ]; then - echo "TEST FAILED" - rc=151 - exit $rc - fi - -##### Populate cache integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - POPULATE CACHE" - rm -rf tmp - mkdir tmp - mkdir tmp/cache - set +e - $0 -c ${BUILD_CACHE_URI} -f PD tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=152 - exit $rc - fi - - if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then - echo "TEST FAILED" - rc=153 - exit $rc - fi - rc=154 - cache_check - -##### Build uinge cache integration test #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - BUILD USING CACHE" - set +e - $0 -c ${BUILD_CACHE_URI} -f D tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=155 - exit $rc - fi - - if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then - echo "TEST FAILED" - rc=156 - exit $rc - fi - rc=157 - cache_check - -#### Repopulate cache because cache non existant #### - make -C ${BUILD_BASE} clean - echo "TEST - $0 - BUILD USING CACHE" - rm -rf tmp/cache/* - set +e - $0 -c ${BUILD_CACHE_URI} -f D tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=158 - exit $rc - fi - - if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then - echo "TEST FAILED" - rc=160 - exit $rc - fi - rc=161 - cache_check - -# Repopulate cache because cach is ivalidated -#TBD -rm -rf tmp -echo "All tests passed!" -rc=0 -exit $rc -} - -#DEFAULT VALUES +############################################################################ +# BEGIN of variables to customize +# BUILD_BASE=$(readlink -e ../build/) RESULT_DIR="${BUILD_BASE}/release" BUILD_SPEC="${BUILD_BASE}/config.mk" @@ -207,8 +89,14 @@ CACHE_DIR="cache" LOCAL_CACHE_ARCH_NAME="fuel-cache" REMOTE_CACHE_ARCH_NAME="fuel_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")" REMOTE_ACCESS_METHD=curl - -#SCRIPT ASSIGNED VARIABLES +INCLUDE_DIR=../include +# +# END of variables to customize +############################################################################ + +############################################################################ +# BEGIN of script assigned variables +# SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) LOCK_FILE="${SCRIPT_DIR}/.build.lck" TEST_SUCCEED=0 @@ -220,6 +108,7 @@ RECURSIV=0 DETACH=0 DEBUG=0 INTEGRATION_TEST=0 +FULL_INTEGRATION_TEST=0 INTERACTIVE=0 BUILD_CACHE_URI= BUILD_SPEC= @@ -227,8 +116,22 @@ BUILD_DIR= BUILD_LOG= BUILD_VERSION= MAKE_ARGS= - -while getopts "s:c:v:f:l:r:RTh" OPTION +# +# END of script assigned variables +############################################################################ + +############################################################################ +# BEGIN of include pragmas +# +source ${INCLUDE_DIR}/build.sh.debug +# +# END of include +############################################################################ + +############################################################################ +# BEGIN of main +# +while getopts "s:c:v:f:l:r:RtTh" OPTION do case $OPTION in h) @@ -264,8 +167,13 @@ do RECURSIVE=1 ;; + t) + INTEGRATION_TEST=1 + ;; + T) INTEGRATION_TEST=1 + FULL_INTEGRATION_TEST=1 ;; *) @@ -316,7 +224,7 @@ for ((i=0; i<${#BUILD_FLAGS};i++)); do ;; *) - echo "${BUILD_FLAGS:$i:1} is not a valid build flag" + echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...." rc=100 exit $rc ;; @@ -330,13 +238,13 @@ if [ ${INTEGRATION_TEST} -eq 1 ]; then fi if [ ! -f ${BUILD_SPEC} ]; then - echo "spec file does not exist: $BUILD_SPEC" + echo "spec file does not exist: $BUILD_SPEC - exiting ...." rc=100 exit $rc fi if [ -z ${BUILD_DIR} ]; then - echo "Missing build directory" + echo "Missing build directory - exiting ...." rc=100 exit $rc fi @@ -379,7 +287,7 @@ if [ ! -z ${BUILD_CACHE_URI} ]; then mkdir /tmp/cache echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..." set +e - ${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} + ${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz rc=$? set -e if [ $rc -ne 0 ]; then @@ -389,10 +297,11 @@ if [ ! -z ${BUILD_CACHE_URI} ]; then echo "Unpacking cache file ..." tar -C /tmp/cache -xvf /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz cp /tmp/cache/cache/.versions ${BUILD_BASE}/. - echo "Validating cache content ..." set +e make -C ${BUILD_BASE} validate-cache; rc=$? + set -e + if [ $rc -ne 0 ]; then echo "Cache invalid - a new cache will be built " POPULATE_CACHE=1 @@ -455,13 +364,13 @@ rc=$? set -e if [ $rc -gt 0 ]; then - echo "Build: make prepare-cache failed, exiting ..." + echo "Build: make prepare-cache failed - exiting ..." rc=100 exit $rc fi echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..." rm -rf ${BUILD_DIR} -mkdir ${BUILD_DIR} +mkdir -p ${BUILD_DIR} cp ${BUILD_BASE}/.versions ${BUILD_DIR} cp ${RESULT_DIR}/*.iso* ${BUILD_DIR} @@ -470,9 +379,12 @@ if [ $POPULATE_CACHE -eq 1 ]; then echo "Building cache ..." tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}" - ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} + ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz fi fi echo "Success!!!" exit 0 +# +# END of main +############################################################################ diff --git a/fuel/include/build.sh.debug b/fuel/include/build.sh.debug new file mode 100644 index 0000000..b6594cb --- /dev/null +++ b/fuel/include/build.sh.debug @@ -0,0 +1,355 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +############################################################################ +# BEGIN of build script integration test +# +integration-test () { + +test_clean () { + make -C ${BUILD_BASE} clean + test_cache_clean + test_output_clean +} + +test_cache_clean () { + rm -rf ${TEST_BUILD_CACHE_URI_DIR} +} + +test_output_clean () { + rm -rf ${TEST_BUILD_RESULT_DIR} +} + +test_prepare () { + test_clean + test_cache_clean + test_output_clean + mkdir -p ${TEST_BUILD_CACHE_URI_DIR} +} + +test_cache_check () { + if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \ + [ ! -f ${BUILD_BASE}/.versions ] || \ + [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \ + [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then + echo "Cache not present in the build system" + echo "TEST FAILED" + exit $rc + fi + + if [ ! -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ]; then + echo "Cache archive not present" + echo "TEST FAILED" + exit $rc + fi + + if tar -tf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}/.mark > /dev/null 2>&1; then + echo "Cache archive is invalidated" + echo "TEST FAILED" + exit $rc + fi +} + +test_output_check () { + if [ ! -f ${TEST_BUILD_RESULT_DIR}/*.iso ] || \ + [ ! -f ${TEST_BUILD_RESULT_DIR}/*.txt ] || \ + [ ! -f ${TEST_BUILD_RESULT_DIR}/.versions ]; then + echo "Build output not produced by the build system" + echo "TEST FAILED" + exit $rc + fi +} + +test_invalidate_cache () { + tar -C ${TEST_BUILD_CACHE_URI_DIR} -xf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz + echo "cache.mk 123" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions + echo "config.mk 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions + echo "fuel 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions + echo "odl 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions + + echo "This file marks the invalid cache" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.mark + rm -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz + tar --dereference -C ${TEST_BUILD_CACHE_URI_DIR} -caf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} +} + +##### Always succeed integration test #### +build_tc1 () { + echo "TEST - $0 - ALWAYS SUCCEED" + set +e + $0 -f s tmp/output + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "TEST FAILED" + rc=151 + exit $rc + fi +} + +##### Always fail integration test (TC2) #### +build_tc2 () { + echo "TEST - $0 - ALWAYS FAIL" + set +e + $0 -f f tmp/output + rc=$? + set -e + if [ $rc -eq 0 ]; then + echo "TEST FAILED" + rc=152 + exit $rc + fi +} + +##### Fake build and populate dummy cache - integration test (TC3) #### +build_tc3 () { + echo "TEST - $0 - FAKE BUILD AND POPULATE DUMMY CACHE" + rm -rf tmp + mkdir -p ${TEST_BUILD_CACHE_URI_DIR} + set +e + $0 -c ${TEST_BUILD_CACHE_URI} -f PD ${TEST_BUILD_RESULT_DIR} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build failed" + echo "TEST FAILED" + rc=152 + exit $rc + fi + + rc=153 + test_output_check + + rc=154 + test_cache_check +} + +##### Fake build usng dummy cache integration test (TC4) #### +build_tc4 () { + echo "TEST - $0 - FAKE BUILD USING DUMMY CACHE" + set +e + $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build failed" + echo "TEST FAILED" + rc=155 + exit $rc + fi + + rc=156 + test_output_check + + rc=157 + test_cache_check +} + +#### Fake build and repopulate cache because cache non existant (TC5) #### +build_tc5 () { + echo "TEST - $0 - FAKE BUILD AND REPOPULATE CACHE BECAUSE NON EXISTANT" + rm -rf ${TEST_BUILD_CACHE_URI_DIR}/* + set +e + $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build failed" + echo "TEST FAILED" + rc=158 + exit $rc + fi + + rc=159 + test_output_check + + rc=160 + test_cache_check +} + +#### Fake build and repopulate cache because cache is ivalidated (TC6) #### +build_tc6 () { + echo "TEST - $0 - FAKE BUILD AND REPOPULATE DUE TO CACHE INVALIDATION" + test_invalidate_cache + set +e + $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build failed" + echo "TEST FAILED" + rc=161 + exit $rc + fi + + rc=162 + test_output_check + + rc=163 + test_cache_check +} + +#### Real build and populate cache (TC7) #### +build_tc7 () { + echo "TEST - $0 - FULL BUILD AND POPULATE CACHE" + test_prepare + BUILD_START=$(date +%s) + set +e + $0 -c ${TEST_BUILD_CACHE_URI} -f Pi ${TEST_BUILD_RESULT_DIR} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build failed" + echo "TEST FAILED" + rc=164 + exit $rc + fi + + BUILD_TIME=$(($(date +%s)-${BUILD_START})) + rc=165 + test_output_check + rc=166 + test_cache_check +} + +#### Real build using cache from TC7 (TC8) #### +build_tc8 () { + echo "TEST - $0 - INCREMENTAL BUILD AND USING CACHE" + BUILD_START=$(date +%s) + set +e + $0 -c ${TEST_BUILD_CACHE_URI} -f i ${TEST_BUILD_RESULT_DIR} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build failed" + echo "TEST FAILED" + rc=167 + exit $rc + fi + BUILD_TIME=$(($(date +%s)-${BUILD_START})) + test_output_check + rc=168 + test_cache_check + rc=169 +} + +TEST_BUILD_ARTIFACT_DIR=${BUILD_BASE}/test +TEST_BUILD_CACHE_URI_DIR=${TEST_BUILD_ARTIFACT_DIR} +TEST_BUILD_CACHE_URI=file://${TEST_BUILD_CACHE_URI_DIR} +TEST_BUILD_RESULT_DIR=${BUILD_BASE}/tmp/output + +TEST_START=$(date +%s) +# Run test case 1: Always succeed integration test +echo ========================================================================== +echo ========================= RUNNING TEST CASE 1 ============================ +echo ========================================================================== +test_prepare +build_tc1 + +# Run test case 2: Always fail integration test +echo ========================================================================== +echo ========================= RUNNING TEST CASE 2 ============================ +echo ========================================================================== +test_prepare +build_tc2 + +# Run test case 3: Fake build and populate a build cach +echo ========================================================================== +echo ========================= RUNNING TEST CASE 3 ============================ +echo ========================================================================== +test_prepare +build_tc3 + +# Run test case 4: Fake build using the build cache produced in TC 3 +echo ========================================================================== +echo ========================= RUNNING TEST CASE 4 ============================ +echo ========================================================================== +build_tc4 + +# Run test case 5: Fake build and repopulate the build cache as there is no existing one +echo ========================================================================== +echo ========================= RUNNING TEST CASE 5 ============================ +echo ========================================================================== +test_prepare +build_tc5 + +# Run test case 6: Fake build and repopulate the build cache as there the current one is invalidated/outdated +echo ========================================================================== +echo ========================= RUNNING TEST CASE 6 ============================ +echo ========================================================================== +test_invalidate_cache +build_tc4 + +# Run test case 7: Build from upstream repos and populate the build cache +if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then +echo ========================================================================== +echo ========================= RUNNING TEST CASE 7 ============================ +echo ========================================================================== + test_prepare + build_tc7 + POPULATE_CACHE_TIME=${BUILD_TIME} +fi + +# Run test case 8: Build from cache produced in TC7 +if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then +echo ========================================================================== +echo ========================= RUNNING TEST CASE 8 ============================ +echo ========================================================================== + build_tc8 + PARTIAL_BUILD_TIME=${BUILD_TIME} +fi + +# Run test case 9: Build and repopulate the cache as the current one is invalidated/not up to date +if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then +echo ========================================================================== +echo ========================= RUNNING TEST CASE 9 ============================ +echo ========================================================================== + echo "TEST - $0 - REPOPULATING CACHE DUE TO CACHE INVALIDATION" + test_invalidate_cache + build_tc8 +fi + +TEST_TIME=$(($(date +%s)-${TEST_START})) + +rm -rf tmp +echo "All tests passed!, test duration was ${TEST_TIME} seconds" +if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then + echo "Full build took ${POPULATE_CACHE_TIME} second" + echo "Incremental build took ${PARTIAL_BUILD_TIME} seconds" +fi +rc=0 +exit $rc +} +# +# END of build script integration test +############################################################################ + +############################################################################ +# BEGIN of debug_make - a fake make initiated with -D for debugging +# +debug_make () { + make -C ${BUILD_BASE} clean + echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso + + echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions + echo "cache.mk" $(md5sum ${BUILD_BASE}/cache.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions + echo "config.mk" $(md5sum ${BUILD_BASE}/config.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions + echo "fuel" $(git ls-remote $(make -C ${BUILD_BASE} get-fuel-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions + echo "odl" $(git ls-remote $(make -C ${BUILD_BASE} get-odl-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions + + rm -rf ${BUILD_BASE}/release + mkdir ${BUILD_BASE}/release + echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso + echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt + echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history + echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log +} +# +# END of debug_make +############################################################################ -- 2.16.6